LCOV - code coverage report
Current view: top level - gcc - tree-vect-loop.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 88.9 % 5252 4668
Test Date: 2026-11-14 16:39:50 Functions: 95.2 % 104 99
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Loop Vectorization
       2              :    Copyright (C) 2003-2026 Free Software Foundation, Inc.
       3              :    Contributed by Dorit Naishlos <dorit@il.ibm.com> and
       4              :    Ira Rosen <irar@il.ibm.com>
       5              : 
       6              : This file is part of GCC.
       7              : 
       8              : GCC is free software; you can redistribute it and/or modify it under
       9              : the terms of the GNU General Public License as published by the Free
      10              : Software Foundation; either version 3, or (at your option) any later
      11              : version.
      12              : 
      13              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      14              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      15              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      16              : for more details.
      17              : 
      18              : You should have received a copy of the GNU General Public License
      19              : along with GCC; see the file COPYING3.  If not see
      20              : <http://www.gnu.org/licenses/>.  */
      21              : 
      22              : #define INCLUDE_ALGORITHM
      23              : #include "config.h"
      24              : #include "system.h"
      25              : #include "coretypes.h"
      26              : #include "backend.h"
      27              : #include "target.h"
      28              : #include "rtl.h"
      29              : #include "tree.h"
      30              : #include "gimple.h"
      31              : #include "cfghooks.h"
      32              : #include "tree-pass.h"
      33              : #include "ssa.h"
      34              : #include "optabs-tree.h"
      35              : #include "memmodel.h"
      36              : #include "optabs.h"
      37              : #include "diagnostic-core.h"
      38              : #include "fold-const.h"
      39              : #include "stor-layout.h"
      40              : #include "cfganal.h"
      41              : #include "gimplify.h"
      42              : #include "gimple-iterator.h"
      43              : #include "gimplify-me.h"
      44              : #include "tree-ssa-loop-ivopts.h"
      45              : #include "tree-ssa-loop-manip.h"
      46              : #include "tree-ssa-loop-niter.h"
      47              : #include "tree-ssa-loop.h"
      48              : #include "cfgloop.h"
      49              : #include "tree-scalar-evolution.h"
      50              : #include "tree-vectorizer.h"
      51              : #include "gimple-fold.h"
      52              : #include "cgraph.h"
      53              : #include "tree-cfg.h"
      54              : #include "tree-if-conv.h"
      55              : #include "internal-fn.h"
      56              : #include "tree-vector-builder.h"
      57              : #include "vec-perm-indices.h"
      58              : #include "tree-eh.h"
      59              : #include "case-cfn-macros.h"
      60              : #include "langhooks.h"
      61              : #include "opts.h"
      62              : #include "hierarchical_discriminator.h"
      63              : 
      64              : /* Loop Vectorization Pass.
      65              : 
      66              :    This pass tries to vectorize loops.
      67              : 
      68              :    For example, the vectorizer transforms the following simple loop:
      69              : 
      70              :         short a[N]; short b[N]; short c[N]; int i;
      71              : 
      72              :         for (i=0; i<N; i++){
      73              :           a[i] = b[i] + c[i];
      74              :         }
      75              : 
      76              :    as if it was manually vectorized by rewriting the source code into:
      77              : 
      78              :         typedef int __attribute__((mode(V8HI))) v8hi;
      79              :         short a[N];  short b[N]; short c[N];   int i;
      80              :         v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
      81              :         v8hi va, vb, vc;
      82              : 
      83              :         for (i=0; i<N/8; i++){
      84              :           vb = pb[i];
      85              :           vc = pc[i];
      86              :           va = vb + vc;
      87              :           pa[i] = va;
      88              :         }
      89              : 
      90              :         The main entry to this pass is vectorize_loops(), in which
      91              :    the vectorizer applies a set of analyses on a given set of loops,
      92              :    followed by the actual vectorization transformation for the loops that
      93              :    had successfully passed the analysis phase.
      94              :         Throughout this pass we make a distinction between two types of
      95              :    data: scalars (which are represented by SSA_NAMES), and memory references
      96              :    ("data-refs").  These two types of data require different handling both
      97              :    during analysis and transformation. The types of data-refs that the
      98              :    vectorizer currently supports are ARRAY_REFS which base is an array DECL
      99              :    (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
     100              :    accesses are required to have a simple (consecutive) access pattern.
     101              : 
     102              :    Analysis phase:
     103              :    ===============
     104              :         The driver for the analysis phase is vect_analyze_loop().
     105              :    It applies a set of analyses, some of which rely on the scalar evolution
     106              :    analyzer (scev) developed by Sebastian Pop.
     107              : 
     108              :         During the analysis phase the vectorizer records some information
     109              :    per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
     110              :    loop, as well as general information about the loop as a whole, which is
     111              :    recorded in a "loop_vec_info" struct attached to each loop.
     112              : 
     113              :    Transformation phase:
     114              :    =====================
     115              :         The loop transformation phase scans all the stmts in the loop, and
     116              :    creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
     117              :    the loop that needs to be vectorized.  It inserts the vector code sequence
     118              :    just before the scalar stmt S, and records a pointer to the vector code
     119              :    in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
     120              :    attached to S).  This pointer will be used for the vectorization of following
     121              :    stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
     122              :    otherwise, we rely on dead code elimination for removing it.
     123              : 
     124              :         For example, say stmt S1 was vectorized into stmt VS1:
     125              : 
     126              :    VS1: vb = px[i];
     127              :    S1:  b = x[i];    STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
     128              :    S2:  a = b;
     129              : 
     130              :    To vectorize stmt S2, the vectorizer first finds the stmt that defines
     131              :    the operand 'b' (S1), and gets the relevant vector def 'vb' from the
     132              :    vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)).  The
     133              :    resulting sequence would be:
     134              : 
     135              :    VS1: vb = px[i];
     136              :    S1:  b = x[i];       STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
     137              :    VS2: va = vb;
     138              :    S2:  a = b;          STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
     139              : 
     140              :         Operands that are not SSA_NAMEs, are data-refs that appear in
     141              :    load/store operations (like 'x[i]' in S1), and are handled differently.
     142              : 
     143              :    Target modeling:
     144              :    =================
     145              :         Currently the only target specific information that is used is the
     146              :    size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
     147              :    Targets that can support different sizes of vectors, for now will need
     148              :    to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".  More
     149              :    flexibility will be added in the future.
     150              : 
     151              :         Since we only vectorize operations which vector form can be
     152              :    expressed using existing tree codes, to verify that an operation is
     153              :    supported, the vectorizer checks the relevant optab at the relevant
     154              :    machine_mode (e.g, optab_handler (add_optab, V8HImode)).  If
     155              :    the value found is CODE_FOR_nothing, then there's no target support, and
     156              :    we can't vectorize the stmt.
     157              : 
     158              :    For additional information on this project see:
     159              :    http://gcc.gnu.org/projects/tree-ssa/vectorization.html
     160              : */
     161              : 
     162              : static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *,
     163              :                                                 unsigned *);
     164              : static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
     165              :                                                gphi **);
     166              : 
     167              : 
     168              : /* Function vect_is_simple_iv_evolution.
     169              : 
     170              :    FORNOW: A simple evolution of an induction variables in the loop is
     171              :    considered a polynomial evolution.  */
     172              : 
     173              : static bool
     174       957179 : vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn,
     175              :                              stmt_vec_info stmt_info)
     176              : {
     177       957179 :   tree init_expr;
     178       957179 :   tree step_expr;
     179       957179 :   tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
     180       957179 :   basic_block bb;
     181              : 
     182              :   /* When there is no evolution in this loop, the evolution function
     183              :      is not "simple".  */
     184       957179 :   if (evolution_part == NULL_TREE)
     185              :     return false;
     186              : 
     187              :   /* When the evolution is a polynomial of degree >= 2
     188              :      the evolution function is not "simple".  */
     189       840903 :   if (tree_is_chrec (evolution_part))
     190              :     return false;
     191              : 
     192       840903 :   step_expr = evolution_part;
     193       840903 :   init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
     194              : 
     195       840903 :   if (dump_enabled_p ())
     196        41890 :     dump_printf_loc (MSG_NOTE, vect_location, "step: %T,  init: %T\n",
     197              :                      step_expr, init_expr);
     198              : 
     199       840903 :   STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
     200       840903 :   STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step_expr;
     201       840903 :   STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_add;
     202              : 
     203       840903 :   if (TREE_CODE (step_expr) != INTEGER_CST
     204        72696 :       && (TREE_CODE (step_expr) != SSA_NAME
     205        57548 :           || ((bb = gimple_bb (SSA_NAME_DEF_STMT (step_expr)))
     206        57292 :               && flow_bb_inside_loop_p (get_loop (cfun, loop_nb), bb))
     207         7779 :           || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
     208          133 :               && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))
     209          133 :                   || !flag_associative_math)))
     210       905917 :       && (TREE_CODE (step_expr) != REAL_CST
     211          416 :           || !flag_associative_math))
     212              :     {
     213        64924 :       if (dump_enabled_p ())
     214         3230 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     215              :                          "step unknown.\n");
     216              :       return false;
     217              :     }
     218              : 
     219              :   return true;
     220              : }
     221              : 
     222              : /* Function vect_is_nonlinear_iv_evolution
     223              : 
     224              :    Only support nonlinear induction for integer type
     225              :    1. neg
     226              :    2. mul by constant
     227              :    3. lshift/rshift by constant.
     228              : 
     229              :    For neg induction, return a fake step as integer -1.  */
     230              : static bool
     231       178745 : vect_is_nonlinear_iv_evolution (class loop* loop, stmt_vec_info stmt_info,
     232              :                                 gphi* loop_phi_node)
     233              : {
     234       178745 :   tree init_expr, ev_expr, result, op1, op2;
     235       178745 :   gimple* def;
     236              : 
     237       178745 :   if (gimple_phi_num_args (loop_phi_node) != 2)
     238              :     return false;
     239              : 
     240       178745 :   init_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_preheader_edge (loop));
     241       178745 :   ev_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_latch_edge (loop));
     242              : 
     243              :   /* Support nonlinear induction only for integer type.  */
     244       178745 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (init_expr)))
     245              :     return false;
     246              : 
     247       111441 :   result = PHI_RESULT (loop_phi_node);
     248              : 
     249       111441 :   if (TREE_CODE (ev_expr) != SSA_NAME
     250       109000 :       || ((def = SSA_NAME_DEF_STMT (ev_expr)), false)
     251       111441 :       || !is_gimple_assign (def))
     252              :     return false;
     253              : 
     254       100518 :   enum tree_code t_code = gimple_assign_rhs_code (def);
     255       100518 :   tree step;
     256       100518 :   switch (t_code)
     257              :     {
     258         3525 :     case NEGATE_EXPR:
     259         3525 :       if (gimple_assign_rhs1 (def) != result)
     260              :         return false;
     261         3525 :       step = build_int_cst (TREE_TYPE (init_expr), -1);
     262         3525 :       STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_neg;
     263         3525 :       break;
     264              : 
     265        12243 :     case RSHIFT_EXPR:
     266        12243 :     case LSHIFT_EXPR:
     267        12243 :     case MULT_EXPR:
     268        12243 :       op1 = gimple_assign_rhs1 (def);
     269        12243 :       op2 = gimple_assign_rhs2 (def);
     270        12243 :       if (TREE_CODE (op2) != INTEGER_CST
     271         8255 :           || op1 != result)
     272              :         return false;
     273         7846 :       step = op2;
     274         7846 :       if (t_code == LSHIFT_EXPR)
     275          536 :         STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shl;
     276         7310 :       else if (t_code == RSHIFT_EXPR)
     277         6315 :         STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shr;
     278              :       /* NEGATE_EXPR and MULT_EXPR are both vect_step_op_mul.  */
     279              :       else
     280          995 :         STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_mul;
     281              :       break;
     282              : 
     283              :     default:
     284              :       return false;
     285              :     }
     286              : 
     287        11371 :   STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
     288        11371 :   STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step;
     289              : 
     290        11371 :   return true;
     291              : }
     292              : 
     293              : /* Returns true if Phi is a first-order recurrence. A first-order
     294              :    recurrence is a non-reduction recurrence relation in which the value of
     295              :    the recurrence in the current loop iteration equals a value defined in
     296              :    the previous iteration.  */
     297              : 
     298              : static bool
     299        71122 : vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo, class loop *loop,
     300              :                                    gphi *phi)
     301              : {
     302              :   /* A nested cycle isn't vectorizable as first order recurrence.  */
     303        71122 :   if (LOOP_VINFO_LOOP (loop_vinfo) != loop)
     304              :     return false;
     305              : 
     306              :   /* Ensure the loop latch definition is from within the loop.  */
     307        70977 :   edge latch = loop_latch_edge (loop);
     308        70977 :   tree ldef = PHI_ARG_DEF_FROM_EDGE (phi, latch);
     309        70977 :   if (TREE_CODE (ldef) != SSA_NAME
     310        68199 :       || SSA_NAME_IS_DEFAULT_DEF (ldef)
     311        68133 :       || is_a <gphi *> (SSA_NAME_DEF_STMT (ldef))
     312       132886 :       || !flow_bb_inside_loop_p (loop, gimple_bb (SSA_NAME_DEF_STMT (ldef))))
     313              :     return false;
     314              : 
     315        61238 :   tree def = gimple_phi_result (phi);
     316              : 
     317              :   /* Ensure every use_stmt of the phi node is dominated by the latch
     318              :      definition.  */
     319        61238 :   imm_use_iterator imm_iter;
     320        61238 :   use_operand_p use_p;
     321        74519 :   FOR_EACH_IMM_USE_FAST (use_p, imm_iter, def)
     322        73880 :     if (!is_gimple_debug (USE_STMT (use_p))
     323       143967 :         && (SSA_NAME_DEF_STMT (ldef) == USE_STMT (use_p)
     324        48480 :             || !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (ldef),
     325              :                                             USE_STMT (use_p))))
     326        60599 :       return false;
     327              : 
     328              :   /* First-order recurrence autovectorization needs shuffle vector.  */
     329          639 :   tree scalar_type = TREE_TYPE (def);
     330          639 :   tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
     331          639 :   if (!vectype)
     332            6 :     return false;
     333              : 
     334              :   return true;
     335              : }
     336              : 
     337              : /* Function vect_analyze_scalar_cycles_1.
     338              : 
     339              :    Examine the cross iteration def-use cycles of scalar variables
     340              :    in LOOP.  LOOP_VINFO represents the loop that is now being
     341              :    considered for vectorization (can be LOOP, or an outer-loop
     342              :    enclosing LOOP).  SLP indicates there will be some subsequent
     343              :    slp analyses or not.  */
     344              : 
     345              : static void
     346       470784 : vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
     347              : {
     348       470784 :   basic_block bb = loop->header;
     349       470784 :   auto_vec<stmt_vec_info, 64> worklist;
     350       470784 :   gphi_iterator gsi;
     351              : 
     352       470784 :   DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
     353              : 
     354              :   /* First - identify all inductions.  Reduction detection assumes that all the
     355              :      inductions have been identified, therefore, this order must not be
     356              :      changed.  */
     357      1678730 :   for (gsi = gsi_start_phis  (bb); !gsi_end_p (gsi); gsi_next (&gsi))
     358              :     {
     359      1207946 :       gphi *phi = gsi.phi ();
     360      1207946 :       tree access_fn = NULL;
     361      1207946 :       tree def = PHI_RESULT (phi);
     362      1207946 :       stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi);
     363              : 
     364              :       /* Skip virtual phi's.  The data dependences that are associated with
     365              :          virtual defs/uses (i.e., memory accesses) are analyzed elsewhere.  */
     366      2415892 :       if (virtual_operand_p (def))
     367       420602 :         continue;
     368              : 
     369              :       /* Skip already analyzed inner loop PHIs of double reductions.  */
     370       958206 :       if (VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
     371         1027 :         continue;
     372              : 
     373       957179 :       if (dump_enabled_p ())
     374        44157 :         dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
     375              :                          (gimple *) phi);
     376              : 
     377       957179 :       STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
     378              : 
     379              :       /* Analyze the evolution function.  */
     380       957179 :       access_fn = analyze_scalar_evolution (loop, def);
     381       957179 :       if (dump_enabled_p ())
     382        44157 :         dump_printf_loc (MSG_NOTE, vect_location,
     383              :                          "Access function of PHI: %T\n", access_fn);
     384       957179 :       if (access_fn)
     385       957179 :         STRIP_NOPS (access_fn);
     386              : 
     387      1127014 :       if ((!access_fn
     388       957179 :            || !vect_is_simple_iv_evolution (loop->num, access_fn, stmt_vinfo)
     389       775979 :            || (LOOP_VINFO_LOOP (loop_vinfo) != loop
     390        11609 :                && (TREE_CODE (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo))
     391              :                    != INTEGER_CST)))
     392              :           /* Only handle nonlinear iv for same loop.  */
     393      1138385 :           && (LOOP_VINFO_LOOP (loop_vinfo) != loop
     394       178745 :               || !vect_is_nonlinear_iv_evolution (loop, stmt_vinfo, phi)))
     395              :         {
     396       169835 :           worklist.safe_push (stmt_vinfo);
     397       169835 :           continue;
     398              :         }
     399              : 
     400       787344 :       gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo)
     401              :                   != NULL_TREE);
     402       787344 :       gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo) != NULL_TREE);
     403              : 
     404       787344 :       if (dump_enabled_p ())
     405        38882 :         dump_printf_loc (MSG_NOTE, vect_location, "Detected induction.\n");
     406       787344 :       STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
     407              : 
     408              :       /* Mark if we have a non-linear IV.  */
     409       787344 :       LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
     410       787344 :         = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add;
     411              :     }
     412              : 
     413              : 
     414              :   /* Second - identify all reductions and nested cycles.  */
     415       640619 :   while (worklist.length () > 0)
     416              :     {
     417       169835 :       stmt_vec_info stmt_vinfo = worklist.pop ();
     418       169835 :       gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
     419       169835 :       tree def = PHI_RESULT (phi);
     420              : 
     421       169835 :       if (dump_enabled_p ())
     422         5275 :         dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
     423              :                          (gimple *) phi);
     424              : 
     425       339670 :       gcc_assert (!virtual_operand_p (def)
     426              :                   && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
     427              : 
     428       169835 :       gphi *double_reduc;
     429       169835 :       stmt_vec_info reduc_stmt_info
     430       169835 :         = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc);
     431       169835 :       if (reduc_stmt_info && double_reduc)
     432              :         {
     433         1129 :           stmt_vec_info inner_phi_info
     434         1129 :               = loop_vinfo->lookup_stmt (double_reduc);
     435              :           /* ???  Pass down flag we're the inner loop of a double reduc.  */
     436         1129 :           stmt_vec_info inner_reduc_info
     437         1129 :             = vect_is_simple_reduction (loop_vinfo, inner_phi_info, NULL);
     438         1129 :           if (inner_reduc_info)
     439              :             {
     440         1027 :               STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
     441         1027 :               STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
     442         1027 :               STMT_VINFO_REDUC_DEF (inner_phi_info) = inner_reduc_info;
     443         1027 :               STMT_VINFO_REDUC_DEF (inner_reduc_info) = inner_phi_info;
     444         1027 :               if (dump_enabled_p ())
     445          130 :                 dump_printf_loc (MSG_NOTE, vect_location,
     446              :                                  "Detected double reduction.\n");
     447              : 
     448         1027 :               STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
     449         1027 :               STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_double_reduction_def;
     450         1027 :               STMT_VINFO_DEF_TYPE (inner_phi_info) = vect_nested_cycle;
     451              :               /* Make it accessible for SLP vectorization.  */
     452         1027 :               LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
     453              :             }
     454          102 :           else if (dump_enabled_p ())
     455           14 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     456              :                              "Unknown def-use cycle pattern.\n");
     457              :         }
     458       168706 :       else if (reduc_stmt_info)
     459              :         {
     460        97584 :           if (loop != LOOP_VINFO_LOOP (loop_vinfo))
     461              :             {
     462         2316 :               if (dump_enabled_p ())
     463          437 :                 dump_printf_loc (MSG_NOTE, vect_location,
     464              :                                  "Detected vectorizable nested cycle.\n");
     465              : 
     466         2316 :               STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
     467              :             }
     468              :           else
     469              :             {
     470        95268 :               STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
     471        95268 :               STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
     472        95268 :               if (dump_enabled_p ())
     473         4120 :                 dump_printf_loc (MSG_NOTE, vect_location,
     474              :                                  "Detected reduction.\n");
     475              : 
     476        95268 :               STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
     477        95268 :               STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def;
     478        95268 :               LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
     479              :             }
     480              :         }
     481        71122 :       else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
     482          633 :         STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
     483              :       else
     484        70489 :         if (dump_enabled_p ())
     485          497 :           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     486              :                            "Unknown def-use cycle pattern.\n");
     487              :     }
     488       470784 : }
     489              : 
     490              : 
     491              : /* Function vect_analyze_scalar_cycles.
     492              : 
     493              :    Examine the cross iteration def-use cycles of scalar variables, by
     494              :    analyzing the loop-header PHIs of scalar variables.  Classify each
     495              :    cycle as one of the following: invariant, induction, reduction, unknown.
     496              :    We do that for the loop represented by LOOP_VINFO, and also to its
     497              :    inner-loop, if exists.
     498              :    Examples for scalar cycles:
     499              : 
     500              :    Example1: reduction:
     501              : 
     502              :               loop1:
     503              :               for (i=0; i<N; i++)
     504              :                  sum += a[i];
     505              : 
     506              :    Example2: induction:
     507              : 
     508              :               loop2:
     509              :               for (i=0; i<N; i++)
     510              :                  a[i] = i;  */
     511              : 
     512              : static void
     513       464870 : vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
     514              : {
     515       464870 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     516              : 
     517       464870 :   vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
     518              : 
     519              :   /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
     520              :      Reductions in such inner-loop therefore have different properties than
     521              :      the reductions in the nest that gets vectorized:
     522              :      1. When vectorized, they are executed in the same order as in the original
     523              :         scalar loop, so we can't change the order of computation when
     524              :         vectorizing them.
     525              :      2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
     526              :         current checks are too strict.  */
     527              : 
     528       464870 :   if (loop->inner)
     529         5914 :     vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
     530       464870 : }
     531              : 
     532              : /* Function vect_get_loop_niters.
     533              : 
     534              :    Determine how many iterations the loop is executed and place it
     535              :    in NUMBER_OF_ITERATIONS.  Place the number of latch iterations
     536              :    in NUMBER_OF_ITERATIONSM1.  Place the condition under which the
     537              :    niter information holds in ASSUMPTIONS.
     538              : 
     539              :    Return the loop exit conditions.  */
     540              : 
     541              : 
     542              : static vec<gcond *>
     543       290813 : vect_get_loop_niters (class loop *loop, const_edge main_exit, tree *assumptions,
     544              :                       tree *number_of_iterations, tree *number_of_iterationsm1)
     545              : {
     546       290813 :   auto_vec<edge> exits = get_loop_exit_edges (loop);
     547       290813 :   vec<gcond *> conds;
     548       581626 :   conds.create (exits.length ());
     549       290813 :   class tree_niter_desc niter_desc;
     550       290813 :   tree niter_assumptions, niter, may_be_zero;
     551              : 
     552       290813 :   *assumptions = boolean_true_node;
     553       290813 :   *number_of_iterationsm1 = chrec_dont_know;
     554       290813 :   *number_of_iterations = chrec_dont_know;
     555              : 
     556       290813 :   DUMP_VECT_SCOPE ("get_loop_niters");
     557              : 
     558       290813 :   if (exits.is_empty ())
     559            0 :     return conds;
     560              : 
     561       290813 :   if (dump_enabled_p ())
     562        15199 :     dump_printf_loc (MSG_NOTE, vect_location, "Loop has %d exits.\n",
     563              :                      exits.length ());
     564              : 
     565       290813 :   edge exit;
     566       290813 :   unsigned int i;
     567       709211 :   FOR_EACH_VEC_ELT (exits, i, exit)
     568              :     {
     569       418398 :       gcond *cond = get_loop_exit_condition (exit);
     570       418398 :       if (cond)
     571       418364 :         conds.safe_push (cond);
     572              : 
     573       418398 :       if (dump_enabled_p ())
     574        16485 :         dump_printf_loc (MSG_NOTE, vect_location, "Analyzing exit %d...\n", i);
     575              : 
     576       418398 :       if (exit != main_exit)
     577       187892 :         continue;
     578              : 
     579       290813 :       may_be_zero = NULL_TREE;
     580       290813 :       if (!number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
     581       290813 :           || chrec_contains_undetermined (niter_desc.niter))
     582        60307 :         continue;
     583              : 
     584       230506 :       niter_assumptions = niter_desc.assumptions;
     585       230506 :       may_be_zero = niter_desc.may_be_zero;
     586       230506 :       niter = niter_desc.niter;
     587              : 
     588       230506 :       if (may_be_zero && integer_zerop (may_be_zero))
     589              :         may_be_zero = NULL_TREE;
     590              : 
     591         9661 :       if (may_be_zero)
     592              :         {
     593         9661 :           if (COMPARISON_CLASS_P (may_be_zero))
     594              :             {
     595              :               /* Try to combine may_be_zero with assumptions, this can simplify
     596              :                  computation of niter expression.  */
     597         9661 :               if (niter_assumptions && !integer_nonzerop (niter_assumptions))
     598          968 :                 niter_assumptions = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
     599              :                                                  niter_assumptions,
     600              :                                                  fold_build1 (TRUTH_NOT_EXPR,
     601              :                                                               boolean_type_node,
     602              :                                                               may_be_zero));
     603              :               else
     604         8693 :                 niter = fold_build3 (COND_EXPR, TREE_TYPE (niter), may_be_zero,
     605              :                                      build_int_cst (TREE_TYPE (niter), 0),
     606              :                                      rewrite_to_non_trapping_overflow (niter));
     607              : 
     608       230506 :               may_be_zero = NULL_TREE;
     609              :             }
     610            0 :           else if (integer_nonzerop (may_be_zero))
     611              :             {
     612            0 :               *number_of_iterationsm1 = build_int_cst (TREE_TYPE (niter), 0);
     613            0 :               *number_of_iterations = build_int_cst (TREE_TYPE (niter), 1);
     614            0 :               continue;
     615              :             }
     616              :           else
     617            0 :             continue;
     618              :        }
     619              : 
     620              :       /* Loop assumptions are based off the normal exit.  */
     621       230506 :       *assumptions = niter_assumptions;
     622       230506 :       *number_of_iterationsm1 = niter;
     623              : 
     624              :       /* We want the number of loop header executions which is the number
     625              :          of latch executions plus one.
     626              :          ???  For UINT_MAX latch executions this number overflows to zero
     627              :          for loops like do { n++; } while (n != 0);  */
     628       230506 :       if (niter && !chrec_contains_undetermined (niter))
     629              :         {
     630       230506 :           niter = fold_build2 (PLUS_EXPR, TREE_TYPE (niter),
     631              :                                unshare_expr (niter),
     632              :                                build_int_cst (TREE_TYPE (niter), 1));
     633       230506 :           if (TREE_CODE (niter) == INTEGER_CST
     634       126756 :               && TREE_CODE (*number_of_iterationsm1) != INTEGER_CST)
     635              :             {
     636              :               /* If we manage to fold niter + 1 into INTEGER_CST even when
     637              :                  niter is some complex expression, ensure back
     638              :                  *number_of_iterationsm1 is an INTEGER_CST as well.  See
     639              :                  PR113210.  */
     640            0 :               *number_of_iterationsm1
     641            0 :                 = fold_build2 (PLUS_EXPR, TREE_TYPE (niter), niter,
     642              :                                build_minus_one_cst (TREE_TYPE (niter)));
     643              :             }
     644              :         }
     645       230506 :       *number_of_iterations = niter;
     646              :     }
     647              : 
     648       290813 :   if (dump_enabled_p ())
     649        15199 :     dump_printf_loc (MSG_NOTE, vect_location, "All loop exits successfully analyzed.\n");
     650              : 
     651       290813 :   return conds;
     652       290813 : }
     653              : 
     654              : /*  Determine the main loop exit for the vectorizer.  */
     655              : 
     656              : edge
     657       503334 : vec_init_loop_exit_info (class loop *loop)
     658              : {
     659              :   /* Before we begin we must first determine which exit is the main one and
     660              :      which are auxiliary exits.  */
     661       503334 :   auto_vec<edge> exits = get_loop_exit_edges (loop);
     662      1001575 :   if (exits.length () == 0)
     663              :     return NULL;
     664       498241 :   if (exits.length () == 1)
     665       325661 :     return exits[0];
     666              : 
     667              :   /* If we have multiple exits, look for counting IV exit.
     668              :      Analyze all exits and return the last one we can analyze.  */
     669       172580 :   class tree_niter_desc niter_desc;
     670       172580 :   edge candidate = NULL;
     671       637727 :   for (edge exit : exits)
     672              :     {
     673       486412 :       if (!get_loop_exit_condition (exit))
     674              :         {
     675        21265 :           if (dump_enabled_p ())
     676           14 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     677              :                              "Unhandled loop exit detected.\n");
     678              :           return NULL;
     679              :         }
     680              : 
     681       465147 :       if (number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
     682       465147 :           && !chrec_contains_undetermined (niter_desc.niter))
     683              :         {
     684       138293 :           tree may_be_zero = niter_desc.may_be_zero;
     685       138293 :           if ((integer_zerop (may_be_zero)
     686              :                /* As we are handling may_be_zero that's not false by
     687              :                   rewriting niter to may_be_zero ? 0 : niter we require
     688              :                   an empty latch.  */
     689       475646 :                || (single_pred_p (loop->latch)
     690        10298 :                    && exit->src == single_pred (loop->latch)
     691         2555 :                    && (integer_nonzerop (may_be_zero)
     692         2555 :                        || COMPARISON_CLASS_P (may_be_zero))))
     693       140848 :               && (!candidate
     694         5842 :                   || dominated_by_p (CDI_DOMINATORS, exit->src,
     695         5842 :                                      candidate->src)))
     696              :             candidate = exit;
     697              :         }
     698              :     }
     699              : 
     700              :   /* If no exit is analyzable by scalar evolution, we return the last exit
     701              :      under the assummption we are dealing with an uncounted loop.  */
     702       208225 :   if (!candidate && single_pred_p (loop->latch))
     703        35645 :     candidate = loop_exits_from_bb_p (loop, single_pred (loop->latch));
     704              : 
     705              :   return candidate;
     706       172580 : }
     707              : 
     708              : /* Function bb_in_loop_p
     709              : 
     710              :    Used as predicate for dfs order traversal of the loop bbs.  */
     711              : 
     712              : static bool
     713      2485886 : bb_in_loop_p (const_basic_block bb, const void *data)
     714              : {
     715      2485886 :   const class loop *const loop = (const class loop *)data;
     716      2485886 :   if (flow_bb_inside_loop_p (loop, bb))
     717              :     return true;
     718              :   return false;
     719              : }
     720              : 
     721              : /* Get and return the basic blocks of LOOP in DFS order.  Store the number of
     722              :    BBs in NBBS.  */
     723              : 
     724              : static basic_block *
     725       440546 : vect_get_loop_dfs_bbs (class loop *loop, unsigned int *nbbs)
     726              : {
     727              :   /* CHECKME: We want to visit all BBs before their successors (except for
     728              :      latch blocks, for which this assertion wouldn't hold).  In the simple
     729              :      case of the loop forms we allow, a dfs order of the BBs would the same
     730              :      as reversed postorder traversal, so we are safe.  */
     731       440546 :   basic_block *bbs = XCNEWVEC (basic_block, loop->num_nodes);
     732       881092 :   *nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p, bbs,
     733       440546 :                               loop->num_nodes, loop);
     734       440546 :   gcc_assert (*nbbs == loop->num_nodes);
     735       440546 :   return bbs;
     736              : }
     737              : 
     738              : /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
     739              :    stmt_vec_info structs for all the stmts in LOOP_IN.  */
     740              : 
     741       541421 : _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared,
     742              :                                 basic_block *bbs_in)
     743              :   : vec_info (vec_info::loop, shared),
     744       541421 :     loop (loop_in),
     745       541421 :     num_itersm1 (NULL_TREE),
     746       541421 :     num_iters (NULL_TREE),
     747       541421 :     num_iters_unchanged (NULL_TREE),
     748       541421 :     num_iters_assumptions (NULL_TREE),
     749       541421 :     vector_costs (nullptr),
     750       541421 :     scalar_costs (nullptr),
     751       541421 :     th (0),
     752       541421 :     versioning_threshold (0),
     753       541421 :     vectorization_factor (0),
     754       541421 :     main_loop_edge (nullptr),
     755       541421 :     skip_main_loop_edge (nullptr),
     756       541421 :     skip_this_loop_edge (nullptr),
     757       541421 :     reusable_accumulators (),
     758       541421 :     suggested_unroll_factor (1),
     759       541421 :     max_vectorization_factor (0),
     760       541421 :     mask_skip_niters (NULL_TREE),
     761       541421 :     mask_skip_niters_pfa_offset (NULL_TREE),
     762       541421 :     rgroup_compare_type (NULL_TREE),
     763       541421 :     simd_if_cond (NULL_TREE),
     764       541421 :     partial_vector_style (vect_partial_vectors_none),
     765       541421 :     unaligned_dr (NULL),
     766       541421 :     peeling_for_alignment (0),
     767       541421 :     ptr_mask (0),
     768       541421 :     max_spec_read_amount (0),
     769       541421 :     nonlinear_iv (false),
     770       541421 :     ivexpr_map (NULL),
     771       541421 :     scan_map (NULL),
     772       541421 :     inner_loop_cost_factor (param_vect_inner_loop_cost_factor),
     773       541421 :     vectorizable (false),
     774       541421 :     can_use_partial_vectors_p (true),
     775       541421 :     must_use_partial_vectors_p (false),
     776       541421 :     using_partial_vectors_p (false),
     777       541421 :     using_decrementing_iv_p (false),
     778       541421 :     using_select_vl_p (false),
     779       541421 :     allow_mutual_alignment (false),
     780       541421 :     partial_load_store_bias (0),
     781       541421 :     peeling_for_gaps (false),
     782       541421 :     peeling_for_niter (false),
     783       541421 :     early_breaks (false),
     784       541421 :     loop_iv_cond (NULL),
     785       541421 :     user_unroll (false),
     786       541421 :     no_data_dependencies (false),
     787       541421 :     has_mask_store (false),
     788       541421 :     scalar_loop_scaling (profile_probability::uninitialized ()),
     789       541421 :     scalar_loop (NULL),
     790       541421 :     main_loop_info (NULL),
     791       541421 :     orig_loop_info (NULL),
     792       541421 :     epilogue_vinfo (NULL),
     793       541421 :     drs_advanced_by (NULL_TREE),
     794       541421 :     vec_loop_main_exit (NULL),
     795       541421 :     vec_epilogue_loop_main_exit (NULL),
     796       541421 :     scalar_loop_main_exit (NULL),
     797       541421 :     early_break_needs_epilogue (false),
     798       541421 :     early_break_niters_var (NULL)
     799              : {
     800       541421 :   nbbs = loop->num_nodes;
     801       541421 :   bbs = XNEWVEC (basic_block, nbbs);
     802       541421 :   memcpy (bbs, bbs_in, nbbs * sizeof (basic_block));
     803              : 
     804      1884413 :   for (unsigned int i = 0; i < nbbs; i++)
     805              :     {
     806      1342992 :       basic_block bb = bbs[i];
     807      1342992 :       gimple_stmt_iterator si;
     808              : 
     809      2768344 :       for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
     810              :         {
     811      1425352 :           gimple *phi = gsi_stmt (si);
     812      1425352 :           gimple_set_uid (phi, 0);
     813      1425352 :           add_stmt (phi);
     814              :         }
     815              : 
     816     12383729 :       for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
     817              :         {
     818      9697745 :           gimple *stmt = gsi_stmt (si);
     819      9697745 :           gimple_set_uid (stmt, 0);
     820      9697745 :           if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
     821      4122416 :             continue;
     822      5575329 :           add_stmt (stmt);
     823              :           /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
     824              :              third argument is the #pragma omp simd if (x) condition, when 0,
     825              :              loop shouldn't be vectorized, when non-zero constant, it should
     826              :              be vectorized normally, otherwise versioned with vectorized loop
     827              :              done if the condition is non-zero at runtime.  */
     828      5575329 :           if (loop_in->simduid
     829        43184 :               && is_gimple_call (stmt)
     830         4245 :               && gimple_call_internal_p (stmt)
     831         4123 :               && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
     832         4122 :               && gimple_call_num_args (stmt) >= 3
     833          104 :               && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
     834      5575433 :               && (loop_in->simduid
     835          104 :                   == SSA_NAME_VAR (gimple_call_arg (stmt, 0))))
     836              :             {
     837          104 :               tree arg = gimple_call_arg (stmt, 2);
     838          104 :               if (integer_zerop (arg) || TREE_CODE (arg) == SSA_NAME)
     839          104 :                 simd_if_cond = arg;
     840              :               else
     841            0 :                 gcc_assert (integer_nonzerop (arg));
     842              :             }
     843              :         }
     844              :     }
     845       541421 : }
     846              : 
     847              : /* Free all levels of rgroup CONTROLS.  */
     848              : 
     849              : void
     850      1391272 : release_vec_loop_controls (vec<rgroup_controls> *controls)
     851              : {
     852      1391272 :   rgroup_controls *rgc;
     853      1391272 :   unsigned int i;
     854      1416497 :   FOR_EACH_VEC_ELT (*controls, i, rgc)
     855        25225 :     rgc->controls.release ();
     856      1391272 :   controls->release ();
     857      1391272 : }
     858              : 
     859              : /* Free all memory used by the _loop_vec_info, as well as all the
     860              :    stmt_vec_info structs of all the stmts in the loop.  */
     861              : 
     862       541421 : _loop_vec_info::~_loop_vec_info ()
     863              : {
     864       541421 :   free (bbs);
     865              : 
     866       541421 :   release_vec_loop_controls (&masks.rgc_vec);
     867       541421 :   release_vec_loop_controls (&lens);
     868       545486 :   delete ivexpr_map;
     869       541743 :   delete scan_map;
     870       541421 :   delete scalar_costs;
     871       541421 :   delete vector_costs;
     872       757891 :   for (auto reduc_info : reduc_infos)
     873       204364 :     delete reduc_info;
     874              : 
     875              :   /* When we release an epiloge vinfo that we do not intend to use
     876              :      avoid clearing AUX of the main loop which should continue to
     877              :      point to the main loop vinfo since otherwise we'll leak that.  */
     878       541421 :   if (loop->aux == this)
     879        64396 :     loop->aux = NULL;
     880      1082842 : }
     881              : 
     882              : /* Return an invariant or register for EXPR and emit necessary
     883              :    computations in the LOOP_VINFO loop preheader.  */
     884              : 
     885              : tree
     886        21048 : cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo, tree expr)
     887              : {
     888        21048 :   if (is_gimple_reg (expr)
     889        21048 :       || is_gimple_min_invariant (expr))
     890              :     return expr;
     891              : 
     892        13731 :   if (! loop_vinfo->ivexpr_map)
     893         4065 :     loop_vinfo->ivexpr_map = new hash_map<tree_operand_hash, tree>;
     894        13731 :   tree &cached = loop_vinfo->ivexpr_map->get_or_insert (expr);
     895        13731 :   if (! cached)
     896              :     {
     897         8880 :       gimple_seq stmts = NULL;
     898         8880 :       cached = force_gimple_operand (unshare_expr (expr),
     899              :                                      &stmts, true, NULL_TREE);
     900         8880 :       if (stmts)
     901              :         {
     902         8726 :           edge e = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
     903         8726 :           gsi_insert_seq_on_edge_immediate (e, stmts);
     904              :         }
     905              :     }
     906        13731 :   return cached;
     907              : }
     908              : 
     909              : /* Return true if we can use CMP_TYPE as the comparison type to produce
     910              :    all masks required to mask LOOP_VINFO.  */
     911              : 
     912              : static bool
     913       113306 : can_produce_all_loop_masks_p (loop_vec_info loop_vinfo, tree cmp_type)
     914              : {
     915       113306 :   rgroup_controls *rgm;
     916       113306 :   unsigned int i;
     917       129063 :   FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
     918       129063 :     if (rgm->type != NULL_TREE
     919       129063 :         && !direct_internal_fn_supported_p (IFN_WHILE_ULT,
     920              :                                             cmp_type, rgm->type,
     921              :                                             OPTIMIZE_FOR_SPEED))
     922              :       return false;
     923              :   return true;
     924              : }
     925              : 
     926              : /* Calculate the maximum number of scalars per iteration for every
     927              :    rgroup in LOOP_VINFO.  */
     928              : 
     929              : static unsigned int
     930        24132 : vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo)
     931              : {
     932        24132 :   unsigned int res = 1;
     933        24132 :   unsigned int i;
     934        24132 :   rgroup_controls *rgm;
     935        57650 :   FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
     936        33518 :     res = MAX (res, rgm->max_nscalars_per_iter);
     937        24132 :   return res;
     938              : }
     939              : 
     940              : /* Calculate the minimum precision necessary to represent:
     941              : 
     942              :       MAX_NITERS * FACTOR
     943              : 
     944              :    as an unsigned integer, where MAX_NITERS is the maximum number of
     945              :    loop header iterations for the original scalar form of LOOP_VINFO.  */
     946              : 
     947              : unsigned
     948        26668 : vect_min_prec_for_max_niters (loop_vec_info loop_vinfo, unsigned int factor)
     949              : {
     950        26668 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     951              : 
     952              :   /* Get the maximum number of iterations that is representable
     953              :      in the counter type.  */
     954        26668 :   tree ni_type;
     955        26668 :   if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
     956        26668 :     ni_type = TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo));
     957              :   else
     958            0 :     ni_type = sizetype;
     959        26668 :   widest_int max_ni = wi::to_widest (TYPE_MAX_VALUE (ni_type)) + 1;
     960              : 
     961              :   /* Get a more refined estimate for the number of iterations.  */
     962        26668 :   widest_int max_back_edges;
     963        26668 :   if (max_loop_iterations (loop, &max_back_edges))
     964        26668 :     max_ni = wi::smin (max_ni, max_back_edges + 1);
     965              : 
     966              :   /* Work out how many bits we need to represent the limit.  */
     967        26668 :   return wi::min_precision (max_ni * factor, UNSIGNED);
     968        26668 : }
     969              : 
     970              : /* True if the loop needs peeling or partial vectors when vectorized.  */
     971              : 
     972              : static bool
     973       160574 : vect_need_peeling_or_partial_vectors_p (loop_vec_info loop_vinfo)
     974              : {
     975       160574 :   unsigned HOST_WIDE_INT const_vf;
     976              : 
     977       160574 :   if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
     978              :     return true;
     979              : 
     980        13971 :   loop_vec_info main_loop_vinfo
     981       159259 :     = (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
     982       159259 :        ? LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) : loop_vinfo);
     983       159259 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
     984        81825 :       && LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo) >= 0)
     985              :     {
     986              :       /* Work out the (constant) number of iterations that need to be
     987              :          peeled for reasons other than niters.  */
     988        81774 :       unsigned int peel_niter
     989              :         = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
     990        81774 :       return !multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter,
     991        81774 :                           LOOP_VINFO_VECT_FACTOR (loop_vinfo));
     992              :     }
     993              : 
     994        77485 :   if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo)
     995        77485 :       && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&const_vf))
     996              :     {
     997              :       /* When the number of iterations is a multiple of the vectorization
     998              :          factor and we are not doing prologue or forced epilogue peeling
     999              :          the epilogue isn't necessary.  */
    1000        77058 :       if (tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
    1001       154116 :           >= (unsigned) exact_log2 (const_vf))
    1002         1802 :         return false;
    1003              :     }
    1004              : 
    1005              :   return true;
    1006              : }
    1007              : 
    1008              : /* Each statement in LOOP_VINFO can be masked where necessary.  Check
    1009              :    whether we can actually generate the masks required.  Return true if so,
    1010              :    storing the type of the scalar IV in LOOP_VINFO_RGROUP_COMPARE_TYPE.  */
    1011              : 
    1012              : static bool
    1013        24132 : vect_verify_full_masking (loop_vec_info loop_vinfo)
    1014              : {
    1015        24132 :   unsigned int min_ni_width;
    1016              : 
    1017              :   /* Use a normal loop if there are no statements that need masking.
    1018              :      This only happens in rare degenerate cases: it means that the loop
    1019              :      has no loads, no stores, and no live-out values.  */
    1020        24132 :   if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
    1021              :     return false;
    1022              : 
    1023              :   /* Produce the rgroup controls.  */
    1024        59421 :   for (auto mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
    1025              :     {
    1026        35289 :       vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    1027        35289 :       tree vectype = mask.first;
    1028        35289 :       unsigned nvectors = mask.second;
    1029              : 
    1030        46446 :       if (masks->rgc_vec.length () < nvectors)
    1031        26328 :         masks->rgc_vec.safe_grow_cleared (nvectors, true);
    1032        35289 :       rgroup_controls *rgm = &(*masks).rgc_vec[nvectors - 1];
    1033              :       /* The number of scalars per iteration and the number of vectors are
    1034              :          both compile-time constants.  */
    1035        35289 :       unsigned int nscalars_per_iter
    1036        35289 :           = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
    1037        35289 :                        LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
    1038              : 
    1039        35289 :       if (rgm->max_nscalars_per_iter < nscalars_per_iter)
    1040              :         {
    1041        28206 :           rgm->max_nscalars_per_iter = nscalars_per_iter;
    1042        28206 :           rgm->type = truth_type_for (vectype);
    1043        28206 :           rgm->factor = 1;
    1044              :         }
    1045              :     }
    1046              : 
    1047        24132 :   unsigned int max_nscalars_per_iter
    1048        24132 :     = vect_get_max_nscalars_per_iter (loop_vinfo);
    1049              : 
    1050              :   /* Work out how many bits we need to represent the limit.  */
    1051        24132 :   min_ni_width
    1052        24132 :     = vect_min_prec_for_max_niters (loop_vinfo, max_nscalars_per_iter);
    1053              : 
    1054              :   /* Find a scalar mode for which WHILE_ULT is supported.  */
    1055        24132 :   opt_scalar_int_mode cmp_mode_iter;
    1056        24132 :   tree cmp_type = NULL_TREE;
    1057        24132 :   tree iv_type = NULL_TREE;
    1058        24132 :   widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
    1059        24132 :   unsigned int iv_precision = UINT_MAX;
    1060              : 
    1061        24132 :   if (iv_limit != -1)
    1062        24132 :     iv_precision = wi::min_precision (iv_limit * max_nscalars_per_iter,
    1063              :                                       UNSIGNED);
    1064              : 
    1065       193056 :   FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
    1066              :     {
    1067       168924 :       unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
    1068       168924 :       if (cmp_bits >= min_ni_width
    1069       168924 :           && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
    1070              :         {
    1071       113306 :           tree this_type = build_nonstandard_integer_type (cmp_bits, true);
    1072       113306 :           if (this_type
    1073       113306 :               && can_produce_all_loop_masks_p (loop_vinfo, this_type))
    1074              :             {
    1075              :               /* Although we could stop as soon as we find a valid mode,
    1076              :                  there are at least two reasons why that's not always the
    1077              :                  best choice:
    1078              : 
    1079              :                  - An IV that's Pmode or wider is more likely to be reusable
    1080              :                    in address calculations than an IV that's narrower than
    1081              :                    Pmode.
    1082              : 
    1083              :                  - Doing the comparison in IV_PRECISION or wider allows
    1084              :                    a natural 0-based IV, whereas using a narrower comparison
    1085              :                    type requires mitigations against wrap-around.
    1086              : 
    1087              :                  Conversely, if the IV limit is variable, doing the comparison
    1088              :                  in a wider type than the original type can introduce
    1089              :                  unnecessary extensions, so picking the widest valid mode
    1090              :                  is not always a good choice either.
    1091              : 
    1092              :                  Here we prefer the first IV type that's Pmode or wider,
    1093              :                  and the first comparison type that's IV_PRECISION or wider.
    1094              :                  (The comparison type must be no wider than the IV type,
    1095              :                  to avoid extensions in the vector loop.)
    1096              : 
    1097              :                  ??? We might want to try continuing beyond Pmode for ILP32
    1098              :                  targets if CMP_BITS < IV_PRECISION.  */
    1099            0 :               iv_type = this_type;
    1100            0 :               if (!cmp_type || iv_precision > TYPE_PRECISION (cmp_type))
    1101              :                 cmp_type = this_type;
    1102            0 :               if (cmp_bits >= GET_MODE_BITSIZE (Pmode))
    1103              :                 break;
    1104              :             }
    1105              :         }
    1106              :     }
    1107              : 
    1108        24132 :   if (!cmp_type)
    1109              :     {
    1110        24132 :       LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.release ();
    1111        24132 :       return false;
    1112              :     }
    1113              : 
    1114            0 :   LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
    1115            0 :   LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
    1116            0 :   LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_while_ult;
    1117            0 :   return true;
    1118        24132 : }
    1119              : 
    1120              : /* Each statement in LOOP_VINFO can be masked where necessary.  Check
    1121              :    whether we can actually generate AVX512 style masks.  Return true if so,
    1122              :    storing the type of the scalar IV in LOOP_VINFO_RGROUP_IV_TYPE.  */
    1123              : 
    1124              : static bool
    1125        24132 : vect_verify_full_masking_avx512 (loop_vec_info loop_vinfo)
    1126              : {
    1127              :   /* Produce differently organized rgc_vec and differently check
    1128              :      we can produce masks.  */
    1129              : 
    1130              :   /* Use a normal loop if there are no statements that need masking.
    1131              :      This only happens in rare degenerate cases: it means that the loop
    1132              :      has no loads, no stores, and no live-out values.  */
    1133        24132 :   if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
    1134              :     return false;
    1135              : 
    1136              :   /* For the decrementing IV we need to represent all values in
    1137              :      [0, niter + niter_skip] where niter_skip is the elements we
    1138              :      skip in the first iteration for prologue peeling.  */
    1139        24132 :   tree iv_type = NULL_TREE;
    1140        24132 :   widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
    1141        24132 :   unsigned int iv_precision = UINT_MAX;
    1142        24132 :   if (iv_limit != -1)
    1143        24132 :     iv_precision = wi::min_precision (iv_limit, UNSIGNED);
    1144              : 
    1145              :   /* First compute the type for the IV we use to track the remaining
    1146              :      scalar iterations.  */
    1147        24132 :   opt_scalar_int_mode cmp_mode_iter;
    1148        31474 :   FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
    1149              :     {
    1150        31474 :       unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
    1151        31474 :       if (cmp_bits >= iv_precision
    1152        31474 :           && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
    1153              :         {
    1154        24132 :           iv_type = build_nonstandard_integer_type (cmp_bits, true);
    1155        24132 :           if (iv_type)
    1156              :             break;
    1157              :         }
    1158              :     }
    1159        24132 :   if (!iv_type)
    1160              :     return false;
    1161              : 
    1162        24132 :   bool ok = true;
    1163              : 
    1164              :   /* Produce the rgroup controls.  */
    1165        51698 :   for (auto const &mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
    1166              :     {
    1167        33720 :       vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    1168        33720 :       tree vectype = mask.first;
    1169        33720 :       unsigned nvectors = mask.second;
    1170              : 
    1171              :       /* The number of scalars per iteration and the number of vectors are
    1172              :          both compile-time constants.  */
    1173        33720 :       unsigned int nscalars_per_iter
    1174        33720 :         = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
    1175        33720 :                      LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
    1176              : 
    1177              :       /* We index the rgroup_controls vector with nscalars_per_iter
    1178              :          which we keep constant and instead have a varying nvectors,
    1179              :          remembering the vector mask with the fewest nV.  */
    1180        43308 :       if (masks->rgc_vec.length () < nscalars_per_iter)
    1181        24165 :         masks->rgc_vec.safe_grow_cleared (nscalars_per_iter, true);
    1182        33720 :       rgroup_controls *rgm = &(*masks).rgc_vec[nscalars_per_iter - 1];
    1183              : 
    1184              :       /* For now vect_get_loop_mask only supports integer mode masks
    1185              :          when we need to split it.  */
    1186        33720 :       tree mask_type = truth_type_for (vectype);
    1187        33720 :       if (GET_MODE_CLASS (TYPE_MODE (mask_type)) != MODE_INT
    1188        33720 :           || TYPE_PRECISION (TREE_TYPE (mask_type)) != 1)
    1189              :         {
    1190         6154 :           ok = false;
    1191         6154 :           break;
    1192              :         }
    1193              : 
    1194        27566 :       if (!rgm->type || rgm->factor > nvectors)
    1195              :         {
    1196        19701 :           rgm->type = mask_type;
    1197        19701 :           rgm->compare_type = NULL_TREE;
    1198        19701 :           rgm->max_nscalars_per_iter = nscalars_per_iter;
    1199        19701 :           rgm->factor = nvectors;
    1200        19701 :           rgm->bias_adjusted_ctrl = NULL_TREE;
    1201              :         }
    1202              :     }
    1203        24132 :   if (!ok)
    1204              :     {
    1205         6154 :       release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
    1206         6154 :       return false;
    1207              :     }
    1208              : 
    1209              :   /* There is no fixed compare type we are going to use but we have to
    1210              :      be able to get at one for each mask group.  */
    1211        17978 :   unsigned int min_ni_width
    1212        17978 :     = wi::min_precision (vect_max_vf (loop_vinfo), UNSIGNED);
    1213              : 
    1214        72507 :   for (auto &rgc : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
    1215              :     {
    1216        18577 :       tree mask_type = rgc.type;
    1217        18577 :       if (!mask_type)
    1218          520 :         continue;
    1219              : 
    1220              :       /* If iv_type is usable as compare type use that - we can elide the
    1221              :          saturation in that case.   */
    1222        18057 :       if (TYPE_PRECISION (iv_type) >= min_ni_width)
    1223              :         {
    1224        18057 :           tree cmp_vectype
    1225        18057 :             = build_vector_type (iv_type, TYPE_VECTOR_SUBPARTS (mask_type));
    1226        18057 :           if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
    1227         6032 :             rgc.compare_type = cmp_vectype;
    1228              :         }
    1229        18057 :       if (!rgc.compare_type)
    1230        34554 :         FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
    1231              :           {
    1232        34550 :             unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
    1233        34550 :             if (cmp_bits >= min_ni_width
    1234        34550 :                 && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
    1235              :               {
    1236        34538 :                 tree cmp_type = build_nonstandard_integer_type (cmp_bits, true);
    1237        34538 :                 if (!cmp_type)
    1238            0 :                   continue;
    1239              : 
    1240              :                 /* Check whether we can produce the mask with cmp_type.  */
    1241        34538 :                 tree cmp_vectype
    1242        34538 :                   = build_vector_type (cmp_type, TYPE_VECTOR_SUBPARTS (mask_type));
    1243        34538 :                 if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
    1244              :                   {
    1245        12021 :                     rgc.compare_type = cmp_vectype;
    1246        12021 :                     break;
    1247              :                   }
    1248              :               }
    1249              :         }
    1250        18057 :       if (!rgc.compare_type)
    1251              :         {
    1252              :           ok = false;
    1253              :           break;
    1254              :         }
    1255              :     }
    1256        17978 :   if (!ok)
    1257              :     {
    1258            4 :       release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
    1259            4 :       return false;
    1260              :     }
    1261              : 
    1262        17974 :   LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = error_mark_node;
    1263        17974 :   LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
    1264        17974 :   LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_avx512;
    1265        17974 :   return true;
    1266        24132 : }
    1267              : 
    1268              : /* Check whether we can use vector access with length based on precision
    1269              :    comparison.  So far, to keep it simple, we only allow the case that the
    1270              :    precision of the target supported length is larger than the precision
    1271              :    required by loop niters.  */
    1272              : 
    1273              : static bool
    1274            6 : vect_verify_loop_lens (loop_vec_info loop_vinfo)
    1275              : {
    1276            6 :   if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
    1277              :     return false;
    1278              : 
    1279            0 :   if (!VECTOR_MODE_P (loop_vinfo->vector_mode))
    1280              :     return false;
    1281              : 
    1282            0 :   machine_mode len_load_mode, len_store_mode;
    1283            0 :   if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
    1284            0 :          .exists (&len_load_mode))
    1285            0 :     return false;
    1286            0 :   if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
    1287            0 :          .exists (&len_store_mode))
    1288            0 :     return false;
    1289              : 
    1290            0 :   signed char partial_load_bias = internal_len_load_store_bias
    1291            0 :     (IFN_LEN_LOAD, len_load_mode);
    1292              : 
    1293            0 :   signed char partial_store_bias = internal_len_load_store_bias
    1294            0 :     (IFN_LEN_STORE, len_store_mode);
    1295              : 
    1296            0 :   gcc_assert (partial_load_bias == partial_store_bias);
    1297              : 
    1298            0 :   if (partial_load_bias == VECT_PARTIAL_BIAS_UNSUPPORTED)
    1299              :     return false;
    1300              : 
    1301              :   /* If the backend requires a bias of -1 for LEN_LOAD, we must not emit
    1302              :      len_loads with a length of zero.  In order to avoid that we prohibit
    1303              :      more than one loop length here.  */
    1304            0 :   if (partial_load_bias == -1
    1305            0 :       && LOOP_VINFO_LENS (loop_vinfo).length () > 1)
    1306              :     return false;
    1307              : 
    1308            0 :   LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) = partial_load_bias;
    1309              : 
    1310            0 :   unsigned int max_nitems_per_iter = 1;
    1311            0 :   unsigned int i;
    1312            0 :   rgroup_controls *rgl;
    1313              :   /* Find the maximum number of items per iteration for every rgroup.  */
    1314            0 :   FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), i, rgl)
    1315              :     {
    1316            0 :       unsigned nitems_per_iter = rgl->max_nscalars_per_iter * rgl->factor;
    1317            0 :       max_nitems_per_iter = MAX (max_nitems_per_iter, nitems_per_iter);
    1318              :     }
    1319              : 
    1320              :   /* Work out how many bits we need to represent the length limit.  */
    1321            0 :   unsigned int min_ni_prec
    1322            0 :     = vect_min_prec_for_max_niters (loop_vinfo, max_nitems_per_iter);
    1323              : 
    1324              :   /* Now use the maximum of below precisions for one suitable IV type:
    1325              :      - the IV's natural precision
    1326              :      - the precision needed to hold: the maximum number of scalar
    1327              :        iterations multiplied by the scale factor (min_ni_prec above)
    1328              :      - the Pmode precision
    1329              : 
    1330              :      If min_ni_prec is less than the precision of the current niters,
    1331              :      we prefer to still use the niters type.  Prefer to use Pmode and
    1332              :      wider IV to avoid narrow conversions.  */
    1333              : 
    1334            0 :   unsigned int ni_prec
    1335            0 :     = TYPE_PRECISION (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)));
    1336            0 :   min_ni_prec = MAX (min_ni_prec, ni_prec);
    1337            0 :   min_ni_prec = MAX (min_ni_prec, GET_MODE_BITSIZE (Pmode));
    1338              : 
    1339            0 :   tree iv_type = NULL_TREE;
    1340            0 :   opt_scalar_int_mode tmode_iter;
    1341            0 :   FOR_EACH_MODE_IN_CLASS (tmode_iter, MODE_INT)
    1342              :     {
    1343            0 :       scalar_mode tmode = tmode_iter.require ();
    1344            0 :       unsigned int tbits = GET_MODE_BITSIZE (tmode);
    1345              : 
    1346              :       /* ??? Do we really want to construct one IV whose precision exceeds
    1347              :          BITS_PER_WORD?  */
    1348            0 :       if (tbits > BITS_PER_WORD)
    1349              :         break;
    1350              : 
    1351              :       /* Find the first available standard integral type.  */
    1352            0 :       if (tbits >= min_ni_prec && targetm.scalar_mode_supported_p (tmode))
    1353              :         {
    1354            0 :           iv_type = build_nonstandard_integer_type (tbits, true);
    1355            0 :           break;
    1356              :         }
    1357              :     }
    1358              : 
    1359            0 :   if (!iv_type)
    1360              :     {
    1361            0 :       if (dump_enabled_p ())
    1362            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1363              :                          "can't vectorize with length-based partial vectors"
    1364              :                          " because there is no suitable iv type.\n");
    1365              :       return false;
    1366              :     }
    1367              : 
    1368            0 :   LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = iv_type;
    1369            0 :   LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
    1370            0 :   LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_len;
    1371              : 
    1372            0 :   return true;
    1373              : }
    1374              : 
    1375              : /* Calculate the cost of one scalar iteration of the loop.  */
    1376              : static void
    1377       386642 : vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
    1378              : {
    1379       386642 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    1380       386642 :   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
    1381       386642 :   int nbbs = loop->num_nodes, i;
    1382              : 
    1383       386642 :   DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
    1384              : 
    1385              :   /* Gather costs for statements in the scalar loop.  */
    1386              : 
    1387      1332258 :   for (i = 0; i < nbbs; i++)
    1388              :     {
    1389       945616 :       gimple_stmt_iterator si;
    1390       945616 :       basic_block bb = bbs[i];
    1391              : 
    1392      7827326 :       for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
    1393              :         {
    1394      5936094 :           gimple *stmt = gsi_stmt (si);
    1395      5936094 :           stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
    1396              : 
    1397      5936094 :           if (!is_gimple_assign (stmt)
    1398              :               && !is_gimple_call (stmt)
    1399              :               && !is_a<gcond *> (stmt))
    1400      2201921 :             continue;
    1401              : 
    1402              :           /* Skip stmts that are not vectorized inside the loop.  */
    1403      3734173 :           stmt_vec_info vstmt_info = vect_stmt_to_vectorize (stmt_info);
    1404      3734173 :           if (!STMT_VINFO_RELEVANT_P (vstmt_info)
    1405      1844116 :               && (!STMT_VINFO_LIVE_P (vstmt_info)
    1406           51 :                   || !VECTORIZABLE_CYCLE_DEF
    1407              :                         (STMT_VINFO_DEF_TYPE (vstmt_info))))
    1408      1844116 :             continue;
    1409              : 
    1410      1890057 :           vect_cost_for_stmt kind;
    1411      1890057 :           if (STMT_VINFO_DATA_REF (stmt_info))
    1412              :             {
    1413       908209 :               if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
    1414              :                kind = scalar_load;
    1415              :              else
    1416       339141 :                kind = scalar_store;
    1417              :             }
    1418       981848 :           else if (vect_nop_conversion_p (stmt_info))
    1419        57632 :             continue;
    1420              :           else
    1421              :             kind = scalar_stmt;
    1422              : 
    1423              :           /* The target cost model applies the inner-loop frequency.  */
    1424      1832425 :           record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
    1425              :                             1, kind, stmt_info, 0, vect_body);
    1426              :         }
    1427              :     }
    1428              : 
    1429              :   /* Now accumulate cost.  */
    1430       386642 :   loop_vinfo->scalar_costs = init_cost (loop_vinfo, true);
    1431       386642 :   add_stmt_costs (loop_vinfo->scalar_costs,
    1432              :                   &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo));
    1433       386642 :   loop_vinfo->scalar_costs->finish_cost (nullptr);
    1434       386642 : }
    1435              : 
    1436              : /* Function vect_analyze_loop_form.
    1437              : 
    1438              :    Verify that certain CFG restrictions hold, including:
    1439              :    - the loop has a pre-header
    1440              :    - the loop has a single entry
    1441              :    - nested loops can have only a single exit.
    1442              :    - the loop exit condition is simple enough
    1443              :    - the number of iterations can be analyzed, i.e, a countable loop.  The
    1444              :      niter could be analyzed under some assumptions.  */
    1445              : 
    1446              : opt_result
    1447       470810 : vect_analyze_loop_form (class loop *loop, gimple *loop_vectorized_call,
    1448              :                         vect_loop_form_info *info)
    1449              : {
    1450       470810 :   DUMP_VECT_SCOPE ("vect_analyze_loop_form");
    1451              : 
    1452       470810 :   edge exit_e = vec_init_loop_exit_info (loop);
    1453       470810 :   if (!exit_e)
    1454        30264 :     return opt_result::failure_at (vect_location,
    1455              :                                    "not vectorized:"
    1456              :                                    " Infinite loop detected.\n");
    1457       440546 :   if (loop_vectorized_call)
    1458              :     {
    1459        25515 :       tree arg = gimple_call_arg (loop_vectorized_call, 1);
    1460        25515 :       class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
    1461        25515 :       edge scalar_exit_e = vec_init_loop_exit_info (scalar_loop);
    1462        25515 :       if (!scalar_exit_e)
    1463            0 :         return opt_result::failure_at (vect_location,
    1464              :                                        "not vectorized:"
    1465              :                                        " could not determine main exit from"
    1466              :                                        " loop with multiple exits.\n");
    1467              :     }
    1468              : 
    1469       440546 :   info->loop_exit = exit_e;
    1470       440546 :   if (dump_enabled_p ())
    1471        16618 :       dump_printf_loc (MSG_NOTE, vect_location,
    1472              :                        "using as main loop exit: %d -> %d [AUX: %p]\n",
    1473        16618 :                        exit_e->src->index, exit_e->dest->index, exit_e->aux);
    1474              : 
    1475              :   /* Check if we have any control flow that doesn't leave the loop.  */
    1476       440546 :   unsigned int nbbs;
    1477       440546 :   info->bbs = vect_get_loop_dfs_bbs (loop, &nbbs);
    1478      1351095 :   for (unsigned i = 0; i < nbbs; i++)
    1479      1028170 :     if (EDGE_COUNT (info->bbs[i]->succs) != 1
    1480      1028170 :         && (EDGE_COUNT (info->bbs[i]->succs) != 2
    1481       644502 :             || !loop_exits_from_bb_p (info->bbs[i]->loop_father,
    1482              :                                       info->bbs[i])))
    1483       117621 :       return opt_result::failure_at (vect_location,
    1484              :                                      "not vectorized:"
    1485              :                                      " unsupported control flow in loop.\n");
    1486              : 
    1487              :   /* Check if we have any control flow that doesn't leave the loop.  */
    1488       324023 :   bool has_phi = false;
    1489       324023 :   for (unsigned i = 0; i < nbbs; i++)
    1490       323558 :     if (!gimple_seq_empty_p (phi_nodes (info->bbs[i])))
    1491              :       {
    1492              :         has_phi = true;
    1493              :         break;
    1494              :       }
    1495       322925 :   if (!has_phi)
    1496          465 :     return opt_result::failure_at (vect_location,
    1497              :                                    "not vectorized:"
    1498              :                                    " no scalar evolution detected in loop.\n");
    1499              : 
    1500              :   /* Different restrictions apply when we are considering an inner-most loop,
    1501              :      vs. an outer (nested) loop.
    1502              :      (FORNOW. May want to relax some of these restrictions in the future).  */
    1503              : 
    1504       322460 :   info->inner_loop_cond = NULL;
    1505       322460 :   if (!loop->inner)
    1506              :     {
    1507              :       /* Inner-most loop.  */
    1508              : 
    1509       302375 :       if (empty_block_p (loop->header))
    1510            0 :         return opt_result::failure_at (vect_location,
    1511              :                                        "not vectorized: empty loop.\n");
    1512              :     }
    1513              :   else
    1514              :     {
    1515        20085 :       class loop *innerloop = loop->inner;
    1516        20085 :       edge entryedge;
    1517              : 
    1518              :       /* Nested loop. We currently require that the loop is doubly-nested,
    1519              :          contains a single inner loop with a single exit to the block
    1520              :          with the single exit condition in the outer loop.
    1521              :          Vectorizable outer-loops look like this:
    1522              : 
    1523              :                         (pre-header)
    1524              :                            |
    1525              :                           header <---+
    1526              :                            |         |
    1527              :                           inner-loop |
    1528              :                            |         |
    1529              :                           tail ------+
    1530              :                            |
    1531              :                         (exit-bb)
    1532              : 
    1533              :          The inner-loop also has the properties expected of inner-most loops
    1534              :          as described above.  */
    1535              : 
    1536        20085 :       if ((loop->inner)->inner || (loop->inner)->next)
    1537         3092 :         return opt_result::failure_at (vect_location,
    1538              :                                        "not vectorized:"
    1539              :                                        " multiple nested loops.\n");
    1540              : 
    1541        16993 :       entryedge = loop_preheader_edge (innerloop);
    1542        16993 :       if (entryedge->src != loop->header
    1543        16462 :           || !single_exit (innerloop)
    1544        28625 :           || single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
    1545         5672 :         return opt_result::failure_at (vect_location,
    1546              :                                        "not vectorized:"
    1547              :                                        " unsupported outerloop form.\n");
    1548              : 
    1549              :       /* Analyze the inner-loop.  */
    1550        11321 :       vect_loop_form_info inner;
    1551        11321 :       opt_result res = vect_analyze_loop_form (loop->inner, NULL, &inner);
    1552        11321 :       if (!res)
    1553              :         {
    1554          417 :           if (dump_enabled_p ())
    1555            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1556              :                              "not vectorized: Bad inner loop.\n");
    1557          417 :           return res;
    1558              :         }
    1559              : 
    1560              :       /* Don't support analyzing niter under assumptions for inner
    1561              :          loop.  */
    1562        10904 :       if (!integer_onep (inner.assumptions))
    1563          263 :         return opt_result::failure_at (vect_location,
    1564              :                                        "not vectorized: Bad inner loop.\n");
    1565              : 
    1566        10641 :       if (inner.number_of_iterations ==  chrec_dont_know
    1567        10641 :           || !expr_invariant_in_loop_p (loop, inner.number_of_iterations))
    1568         1857 :         return opt_result::failure_at (vect_location,
    1569              :                                        "not vectorized: inner-loop count not"
    1570              :                                        " invariant.\n");
    1571              : 
    1572         8784 :       if (dump_enabled_p ())
    1573         1051 :         dump_printf_loc (MSG_NOTE, vect_location,
    1574              :                          "Considering outer-loop vectorization.\n");
    1575         8784 :       info->inner_loop_cond = inner.conds[0];
    1576        11321 :     }
    1577              : 
    1578       311159 :   if (EDGE_COUNT (loop->header->preds) != 2)
    1579            0 :     return opt_result::failure_at (vect_location,
    1580              :                                    "not vectorized:"
    1581              :                                    " too many incoming edges.\n");
    1582              : 
    1583              :   /* We assume that the latch is empty.  */
    1584       311159 :   basic_block latch = loop->latch;
    1585       311159 :   do
    1586              :     {
    1587       311159 :       if (!empty_block_p (latch)
    1588       311159 :           || !gimple_seq_empty_p (phi_nodes (latch)))
    1589        20313 :         return opt_result::failure_at (vect_location,
    1590              :                                        "not vectorized: latch block not "
    1591              :                                        "empty.\n");
    1592       290846 :       latch = single_pred (latch);
    1593              :     }
    1594       581692 :   while (single_succ_p (latch));
    1595              : 
    1596              :   /* Make sure there is no abnormal exit.  */
    1597       290846 :   auto_vec<edge> exits = get_loop_exit_edges (loop);
    1598      1290936 :   for (edge e : exits)
    1599              :     {
    1600       418431 :       if (e->flags & EDGE_ABNORMAL)
    1601           33 :         return opt_result::failure_at (vect_location,
    1602              :                                        "not vectorized:"
    1603              :                                        " abnormal loop exit edge.\n");
    1604              :     }
    1605              : 
    1606       290813 :   info->conds
    1607       290813 :     = vect_get_loop_niters (loop, exit_e, &info->assumptions,
    1608              :                             &info->number_of_iterations,
    1609       290813 :                             &info->number_of_iterationsm1);
    1610       290813 :   if (info->conds.is_empty ())
    1611           34 :     return opt_result::failure_at
    1612           34 :       (vect_location,
    1613              :        "not vectorized: complicated exit condition.\n");
    1614              : 
    1615              :   /* Determine what the primary and alternate exit conds are.  */
    1616       709143 :   for (unsigned i = 0; i < info->conds.length (); i++)
    1617              :     {
    1618       418364 :       gcond *cond = info->conds[i];
    1619       418364 :       if (exit_e->src == gimple_bb (cond))
    1620       290779 :         std::swap (info->conds[0], info->conds[i]);
    1621              :     }
    1622              : 
    1623       290779 :   if (chrec_contains_undetermined (info->number_of_iterations))
    1624              :     {
    1625        60273 :       if (dump_enabled_p ())
    1626          261 :         dump_printf_loc (MSG_NOTE, vect_location,
    1627              :                          "Loop being analyzed as uncounted.\n");
    1628        60273 :       if (loop->inner)
    1629          563 :         return opt_result::failure_at
    1630          563 :           (vect_location,
    1631              :            "not vectorized: outer loop vectorization of uncounted loops"
    1632              :            " is unsupported.\n");
    1633        59710 :       return opt_result::success ();
    1634              :     }
    1635              : 
    1636       230506 :   if (integer_zerop (info->assumptions))
    1637            4 :     return opt_result::failure_at
    1638            4 :       (info->conds[0],
    1639              :        "not vectorized: number of iterations cannot be computed.\n");
    1640              : 
    1641       230502 :   if (integer_zerop (info->number_of_iterations))
    1642           18 :     return opt_result::failure_at
    1643           18 :       (info->conds[0],
    1644              :        "not vectorized: number of iterations = 0.\n");
    1645              : 
    1646       230484 :   if (!(tree_fits_shwi_p (info->number_of_iterations)
    1647       126728 :         && tree_to_shwi (info->number_of_iterations) > 0))
    1648              :     {
    1649       103756 :       if (dump_enabled_p ())
    1650              :         {
    1651         2785 :           dump_printf_loc (MSG_NOTE, vect_location,
    1652              :                            "Symbolic number of iterations is ");
    1653         2785 :           dump_generic_expr (MSG_NOTE, TDF_DETAILS, info->number_of_iterations);
    1654         2785 :           dump_printf (MSG_NOTE, "\n");
    1655              :         }
    1656              :     }
    1657              : 
    1658       230484 :   return opt_result::success ();
    1659       290846 : }
    1660              : 
    1661              : /* Create a loop_vec_info for LOOP with SHARED and the
    1662              :    vect_analyze_loop_form result.  */
    1663              : 
    1664              : loop_vec_info
    1665       541421 : vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
    1666              :                         const vect_loop_form_info *info,
    1667              :                         loop_vec_info orig_loop_info)
    1668              : {
    1669       541421 :   loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared, info->bbs);
    1670       541421 :   LOOP_VINFO_NITERSM1 (loop_vinfo) = info->number_of_iterationsm1;
    1671       541421 :   LOOP_VINFO_NITERS (loop_vinfo) = info->number_of_iterations;
    1672       541421 :   LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = info->number_of_iterations;
    1673       541421 :   LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = orig_loop_info;
    1674       541421 :   if (orig_loop_info && LOOP_VINFO_EPILOGUE_P (orig_loop_info))
    1675          404 :     LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo)
    1676          404 :       = LOOP_VINFO_MAIN_LOOP_INFO (orig_loop_info);
    1677              :   else
    1678       541017 :     LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) = orig_loop_info;
    1679              :   /* Also record the assumptions for versioning.  */
    1680       541421 :   if (!integer_onep (info->assumptions) && !orig_loop_info)
    1681        20348 :     LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = info->assumptions;
    1682              : 
    1683      2404574 :   for (gcond *cond : info->conds)
    1684              :     {
    1685       780311 :       stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
    1686              :       /* Mark the statement as a condition.  */
    1687       780311 :       STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
    1688              :     }
    1689              : 
    1690       541421 :   unsigned cond_id = 0;
    1691       541421 :   if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
    1692       476576 :     LOOP_VINFO_LOOP_IV_COND (loop_vinfo) = info->conds[cond_id++];
    1693              : 
    1694       845156 :   for (; cond_id < info->conds.length (); cond_id ++)
    1695       303735 :     LOOP_VINFO_LOOP_CONDS (loop_vinfo).safe_push (info->conds[cond_id]);
    1696              : 
    1697       541421 :   LOOP_VINFO_MAIN_EXIT (loop_vinfo) = info->loop_exit;
    1698              : 
    1699              :   /* Check to see if we're vectorizing multiple exits.  */
    1700       541421 :   LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
    1701       541421 :     = !LOOP_VINFO_LOOP_CONDS (loop_vinfo).is_empty ();
    1702              : 
    1703              :   /* At the moment we can't support no epilogs for multiple exits, result of
    1704              :      the first compare should be masked by that of the second.  We can only
    1705              :      allow it if the early exits have the same live values.  for differing
    1706              :      values we have to calculate a third mask to disambiguate. */
    1707       541421 :   LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo)
    1708       541421 :     = LOOP_VINFO_LOOP_CONDS (loop_vinfo).length () > 1;
    1709              : 
    1710       541421 :   if (info->inner_loop_cond)
    1711              :     {
    1712              :       /* If we have an estimate on the number of iterations of the inner
    1713              :          loop use that to limit the scale for costing, otherwise use
    1714              :          --param vect-inner-loop-cost-factor literally.  */
    1715         7080 :       widest_int nit;
    1716         7080 :       if (estimated_stmt_executions (loop->inner, &nit))
    1717         5801 :         LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo)
    1718         5801 :           = wi::smin (nit, param_vect_inner_loop_cost_factor).to_uhwi ();
    1719         7080 :     }
    1720              : 
    1721       541421 :   return loop_vinfo;
    1722              : }
    1723              : 
    1724              : 
    1725              : 
    1726              : /* Return true if we know that the iteration count is smaller than the
    1727              :    vectorization factor.  Return false if it isn't, or if we can't be sure
    1728              :    either way.  */
    1729              : 
    1730              : static bool
    1731       159576 : vect_known_niters_smaller_than_vf (loop_vec_info loop_vinfo)
    1732              : {
    1733       159576 :   unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
    1734              : 
    1735       159576 :   HOST_WIDE_INT max_niter;
    1736       159576 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
    1737        82005 :     max_niter = LOOP_VINFO_INT_NITERS (loop_vinfo);
    1738              :   else
    1739        77571 :     max_niter = max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo));
    1740              : 
    1741       159576 :   if (max_niter != -1 && (unsigned HOST_WIDE_INT) max_niter < assumed_vf)
    1742        11296 :     return true;
    1743              : 
    1744              :   return false;
    1745              : }
    1746              : 
    1747              : /* Analyze the cost of the loop described by LOOP_VINFO.  Decide if it
    1748              :    is worthwhile to vectorize.  Return 1 if definitely yes, 0 if
    1749              :    definitely no, or -1 if it's worth retrying.  */
    1750              : 
    1751              : static int
    1752       159590 : vect_analyze_loop_costing (loop_vec_info loop_vinfo,
    1753              :                            unsigned *suggested_unroll_factor)
    1754              : {
    1755       159590 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    1756       159590 :   unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
    1757              : 
    1758              :   /* Only loops that can handle partially-populated vectors can have iteration
    1759              :      counts less than the vectorization factor.  */
    1760       159590 :   if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
    1761       159590 :       && vect_known_niters_smaller_than_vf (loop_vinfo))
    1762              :     {
    1763        11286 :       if (dump_enabled_p ())
    1764          258 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1765              :                          "not vectorized: iteration count smaller than "
    1766              :                          "vectorization factor.\n");
    1767              :       return 0;
    1768              :     }
    1769              : 
    1770              :   /* If we know the number of iterations we can do better, for the
    1771              :      epilogue we can also decide whether the main loop leaves us
    1772              :      with enough iterations, preferring a smaller vector epilog then
    1773              :      also possibly used for the case we skip the vector loop.  */
    1774       148304 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
    1775              :     {
    1776        72019 :       widest_int scalar_niters
    1777        72019 :         = wi::to_widest (LOOP_VINFO_NITERSM1 (loop_vinfo)) + 1;
    1778        72019 :       if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
    1779              :         {
    1780         2855 :           loop_vec_info orig_loop_vinfo
    1781              :             = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
    1782         2855 :           loop_vec_info main_loop_vinfo
    1783              :             = LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo);
    1784         2855 :           unsigned lowest_vf
    1785         2855 :             = constant_lower_bound (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo));
    1786         2855 :           int prolog_peeling = 0;
    1787         2855 :           if (!vect_use_loop_mask_for_alignment_p (main_loop_vinfo))
    1788         2855 :             prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
    1789         2855 :           if (prolog_peeling >= 0
    1790         2855 :               && known_eq (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
    1791              :                            lowest_vf))
    1792              :             {
    1793         5698 :               unsigned gap
    1794         2849 :                 = LOOP_VINFO_PEELING_FOR_GAPS (main_loop_vinfo) ? 1 : 0;
    1795         5698 :               scalar_niters = ((scalar_niters - gap - prolog_peeling)
    1796         5698 :                                % lowest_vf + gap);
    1797              :             }
    1798              :         }
    1799              :       /* Reject vectorizing for a single scalar iteration, even if
    1800              :          we could in principle implement that using partial vectors.
    1801              :          But allow such vectorization if VF == 1 in case we do not
    1802              :          need to peel for gaps (if we need, avoid vectorization for
    1803              :          reasons of code footprint).  */
    1804        72019 :       unsigned peeling_gap = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo);
    1805        72019 :       if (scalar_niters <= peeling_gap + 1
    1806        72019 :           && (assumed_vf > 1 || peeling_gap != 0))
    1807              :         {
    1808          686 :           if (dump_enabled_p ())
    1809          162 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1810              :                              "not vectorized: loop only has a single "
    1811              :                              "scalar iteration.\n");
    1812              :           return 0;
    1813              :         }
    1814              : 
    1815        71333 :       if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    1816              :         {
    1817              :           /* Check that the loop processes at least one full vector.  */
    1818        71320 :           poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    1819        71320 :           if (known_lt (scalar_niters, vf))
    1820              :             {
    1821          401 :               if (dump_enabled_p ())
    1822          347 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1823              :                                  "loop does not have enough iterations "
    1824              :                                  "to support vectorization.\n");
    1825          441 :               return 0;
    1826              :             }
    1827              : 
    1828              :           /* If we need to peel an extra epilogue iteration to handle data
    1829              :              accesses with gaps, check that there are enough scalar iterations
    1830              :              available.
    1831              : 
    1832              :              The check above is redundant with this one when peeling for gaps,
    1833              :              but the distinction is useful for diagnostics.  */
    1834        70919 :           if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
    1835        71240 :               && known_le (scalar_niters, vf))
    1836              :             {
    1837           40 :               if (dump_enabled_p ())
    1838            9 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1839              :                                  "loop does not have enough iterations "
    1840              :                                  "to support peeling for gaps.\n");
    1841              :               return 0;
    1842              :             }
    1843              :         }
    1844        72019 :     }
    1845              : 
    1846              :   /* If using the "very cheap" model. reject cases in which we'd keep
    1847              :      a copy of the scalar code (even if we might be able to vectorize it).  */
    1848       147177 :   if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
    1849       147177 :       && (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
    1850        77313 :           || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
    1851              :     {
    1852          721 :       if (dump_enabled_p ())
    1853            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1854              :                          "some scalar iterations would need to be peeled\n");
    1855              :       return 0;
    1856              :     }
    1857              : 
    1858       146456 :   int min_profitable_iters, min_profitable_estimate;
    1859       146456 :   vect_estimate_min_profitable_iters (loop_vinfo, &min_profitable_iters,
    1860              :                                       &min_profitable_estimate,
    1861              :                                       suggested_unroll_factor);
    1862              : 
    1863       146456 :   if (min_profitable_iters < 0)
    1864              :     {
    1865        24962 :       if (dump_enabled_p ())
    1866           38 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1867              :                          "not vectorized: vectorization not profitable.\n");
    1868        24962 :       if (dump_enabled_p ())
    1869           38 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1870              :                          "not vectorized: vector version will never be "
    1871              :                          "profitable.\n");
    1872              :       return -1;
    1873              :     }
    1874              : 
    1875       121494 :   int min_scalar_loop_bound = (param_min_vect_loop_bound
    1876       121494 :                                * assumed_vf);
    1877              : 
    1878              :   /* Use the cost model only if it is more conservative than user specified
    1879              :      threshold.  */
    1880       121494 :   unsigned int th = (unsigned) MAX (min_scalar_loop_bound,
    1881              :                                     min_profitable_iters);
    1882              : 
    1883       121494 :   LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = th;
    1884              : 
    1885        65426 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
    1886       186920 :       && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
    1887              :     {
    1888          597 :       if (dump_enabled_p ())
    1889            2 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1890              :                          "not vectorized: vectorization not profitable.\n");
    1891          597 :       if (dump_enabled_p ())
    1892            2 :         dump_printf_loc (MSG_NOTE, vect_location,
    1893              :                          "not vectorized: iteration count smaller than user "
    1894              :                          "specified loop bound parameter or minimum profitable "
    1895              :                          "iterations (whichever is more conservative).\n");
    1896              :       return 0;
    1897              :     }
    1898              : 
    1899              :   /* The static profitablity threshold min_profitable_estimate includes
    1900              :      the cost of having to check at runtime whether the scalar loop
    1901              :      should be used instead.  If it turns out that we don't need or want
    1902              :      such a check, the threshold we should use for the static estimate
    1903              :      is simply the point at which the vector loop becomes more profitable
    1904              :      than the scalar loop.  */
    1905       120897 :   if (min_profitable_estimate > min_profitable_iters
    1906        27792 :       && !LOOP_REQUIRES_VERSIONING (loop_vinfo)
    1907        27208 :       && !LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
    1908          644 :       && !LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
    1909       121541 :       && !vect_apply_runtime_profitability_check_p (loop_vinfo))
    1910              :     {
    1911           14 :       if (dump_enabled_p ())
    1912            8 :         dump_printf_loc (MSG_NOTE, vect_location, "no need for a runtime"
    1913              :                          " choice between the scalar and vector loops\n");
    1914           14 :       min_profitable_estimate = min_profitable_iters;
    1915              :     }
    1916              : 
    1917              :   /* If the vector loop needs multiple iterations to be beneficial then
    1918              :      things are probably too close to call, and the conservative thing
    1919              :      would be to stick with the scalar code.  */
    1920       120897 :   if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
    1921        61740 :       && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
    1922       144700 :       && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
    1923              :     {
    1924        15106 :       if (dump_enabled_p ())
    1925          225 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1926              :                          "one iteration of the vector loop would be"
    1927              :                          " more expensive than the equivalent number of"
    1928              :                          " iterations of the scalar loop\n");
    1929              :       return 0;
    1930              :     }
    1931              : 
    1932       105791 :   HOST_WIDE_INT estimated_niter;
    1933              : 
    1934              :   /* If we are vectorizing an epilogue then we know the maximum number of
    1935              :      scalar iterations it will cover is at least one lower than the
    1936              :      vectorization factor of the main loop.  */
    1937       105791 :   if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
    1938        12554 :     estimated_niter
    1939        12554 :       = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
    1940              :   else
    1941              :     {
    1942        93237 :       estimated_niter = estimated_stmt_executions_int (loop);
    1943        93237 :       if (estimated_niter == -1)
    1944        30604 :         estimated_niter = likely_max_stmt_executions_int (loop);
    1945              :     }
    1946        43158 :   if (estimated_niter != -1
    1947       102852 :       && ((unsigned HOST_WIDE_INT) estimated_niter
    1948       102852 :           < MAX (th, (unsigned) min_profitable_estimate)))
    1949              :     {
    1950         4651 :       if (dump_enabled_p ())
    1951           34 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1952              :                          "not vectorized: estimated iteration count too "
    1953              :                          "small.\n");
    1954         4651 :       if (dump_enabled_p ())
    1955           34 :         dump_printf_loc (MSG_NOTE, vect_location,
    1956              :                          "not vectorized: estimated iteration count smaller "
    1957              :                          "than specified loop bound parameter or minimum "
    1958              :                          "profitable iterations (whichever is more "
    1959              :                          "conservative).\n");
    1960              :       return -1;
    1961              :     }
    1962              : 
    1963              :   /* As we cannot use a runtime check to gate profitability for uncounted
    1964              :      loops require either an estimate or if none, at least a profitable
    1965              :      vectorization within the first vector iteration (that condition
    1966              :      will practically never be true due to the required epilog and
    1967              :      likely alignment prologue).   */
    1968       101140 :   if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
    1969          165 :       && estimated_niter == -1
    1970       101278 :       && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
    1971              :     {
    1972          120 :       if (dump_enabled_p ())
    1973            2 :         dump_printf_loc (MSG_NOTE, vect_location,
    1974              :                          "not vectorized: no loop iteration estimate on the "
    1975              :                          "uncounted loop and not trivially profitable.\n");
    1976              :       return -1;
    1977              :     }
    1978              : 
    1979              :   return 1;
    1980              : }
    1981              : 
    1982              : /* Gather data references in LOOP with body BBS and store them into
    1983              :    *DATAREFS.  */
    1984              : 
    1985              : static opt_result
    1986       278593 : vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
    1987              :                            vec<data_reference_p> *datarefs)
    1988              : {
    1989       833233 :   for (unsigned i = 0; i < loop->num_nodes; i++)
    1990      1235674 :     for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
    1991      5417932 :          !gsi_end_p (gsi); gsi_next (&gsi))
    1992              :       {
    1993      4863292 :         gimple *stmt = gsi_stmt (gsi);
    1994      4863292 :         if (is_gimple_debug (stmt))
    1995      2321077 :           continue;
    1996      2542351 :         opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs,
    1997              :                                                         NULL, 0);
    1998      2542351 :         if (!res)
    1999              :           {
    2000        63333 :             if (is_gimple_call (stmt) && loop->safelen)
    2001              :               {
    2002          412 :                 tree fndecl = gimple_call_fndecl (stmt), op;
    2003          412 :                 if (fndecl == NULL_TREE
    2004          412 :                     && gimple_call_internal_p (stmt, IFN_MASK_CALL))
    2005              :                   {
    2006            0 :                     fndecl = gimple_call_arg (stmt, 0);
    2007            0 :                     gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
    2008            0 :                     fndecl = TREE_OPERAND (fndecl, 0);
    2009            0 :                     gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
    2010              :                   }
    2011          412 :                 if (fndecl != NULL_TREE)
    2012              :                   {
    2013          375 :                     cgraph_node *node = cgraph_node::get (fndecl);
    2014          375 :                     if (node != NULL && node->simd_clones != NULL)
    2015              :                       {
    2016          137 :                         unsigned int j, n = gimple_call_num_args (stmt);
    2017          565 :                         for (j = 0; j < n; j++)
    2018              :                           {
    2019          292 :                             op = gimple_call_arg (stmt, j);
    2020          292 :                             if (DECL_P (op)
    2021          292 :                                 || (REFERENCE_CLASS_P (op)
    2022            0 :                                     && get_base_address (op)))
    2023              :                               break;
    2024              :                           }
    2025          137 :                         op = gimple_call_lhs (stmt);
    2026              :                         /* Ignore #pragma omp declare simd functions
    2027              :                            if they don't have data references in the
    2028              :                            call stmt itself.  */
    2029          273 :                         if (j == n
    2030          137 :                             && !(op
    2031          124 :                                  && (DECL_P (op)
    2032          124 :                                      || (REFERENCE_CLASS_P (op)
    2033            0 :                                          && get_base_address (op)))))
    2034          136 :                           continue;
    2035              :                       }
    2036              :                   }
    2037              :               }
    2038        63197 :             return res;
    2039              :           }
    2040              :         /* If dependence analysis will give up due to the limit on the
    2041              :            number of datarefs stop here and fail fatally.  */
    2042      4385150 :         if (datarefs->length ()
    2043      1906132 :             > (unsigned)param_loop_max_datarefs_for_datadeps)
    2044            0 :           return opt_result::failure_at (stmt, "exceeded param "
    2045              :                                          "loop-max-datarefs-for-datadeps\n");
    2046              :       }
    2047       215396 :   return opt_result::success ();
    2048              : }
    2049              : 
    2050              : /* Gather no-wrap bounds from SHARED's datarefs and accumulate them in
    2051              :    one assumption.  Validate this assumption, as well as INFO's niter
    2052              :    assumption and return an error on failure.
    2053              :    If the assumptions are valid, reset loop properties as well as
    2054              :    scev's hashtable and return success.  */
    2055              : 
    2056              : static opt_result
    2057       215396 : vect_build_assumptions (class loop *loop, vec_info_shared *shared,
    2058              :                         vect_loop_form_info *info)
    2059              : {
    2060       215396 :   tree niter = info->number_of_iterations;
    2061       215396 :   bool niter_known = niter && !chrec_contains_undetermined (niter);
    2062              : 
    2063       215396 :   tree nowrap_assum = NULL_TREE;
    2064       215396 :   tree bound_min = NULL_TREE;
    2065              : 
    2066       215396 :   unsigned int i;
    2067       215396 :   struct data_reference *dr;
    2068              : 
    2069              :   /* Build the minimum over all nowrap bounds.  */
    2070       763925 :   FOR_EACH_VEC_ELT (shared->datarefs, i, dr)
    2071              :     {
    2072       548529 :       tree bound = DR_NOWRAP_BOUND (dr);
    2073       548529 :       if (!bound)
    2074       542285 :         continue;
    2075              : 
    2076              :       /* Use scev's function to MIN two bounds.  */
    2077         6244 :       scev_combine_nowrap_bounds (&bound_min, bound);
    2078              :     }
    2079              : 
    2080              :   /* Build "NITER < BOUND_MIN".  */
    2081       215396 :   if (niter_known && bound_min)
    2082              :     {
    2083          979 :       unsigned prec = MAX (TYPE_PRECISION (TREE_TYPE (niter)),
    2084              :                            TYPE_PRECISION (TREE_TYPE (bound_min)));
    2085          979 :       tree t = build_nonstandard_integer_type (prec, 1);
    2086          979 :       nowrap_assum = fold_build2 (LE_EXPR, boolean_type_node,
    2087              :                                   fold_convert (t, niter),
    2088              :                                   fold_convert (t, bound_min));
    2089              : 
    2090              :       /* Try to simplify the assumption with ranger.  */
    2091          979 :       value_range vr (TREE_TYPE (nowrap_assum));
    2092          979 :       tree better_assum;
    2093          979 :       if (get_range_query (cfun)->range_on_edge
    2094          979 :           (vr, loop_preheader_edge (loop), nowrap_assum)
    2095         1958 :           && vr.singleton_p (&better_assum))
    2096          188 :         nowrap_assum = better_assum;
    2097          979 :     }
    2098              : 
    2099              :   /* Combine nowrap assumptions and niter assumption.  */
    2100          979 :   if (nowrap_assum)
    2101              :     {
    2102          979 :       if (dump_enabled_p ())
    2103           16 :         dump_printf_loc (MSG_NOTE, vect_location,
    2104              :                          "adding no-wrap assumption: %T\n", nowrap_assum);
    2105              : 
    2106          979 :       if (info->assumptions && !integer_onep (info->assumptions))
    2107          253 :         nowrap_assum = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
    2108              :                                     info->assumptions, nowrap_assum);
    2109          979 :       info->assumptions = nowrap_assum;
    2110              :     }
    2111              : 
    2112       215396 :   if (integer_zerop (info->assumptions))
    2113          129 :     return opt_result::failure_at
    2114          129 :       (vect_location,
    2115              :        "not vectorized: number of iterations cannot be computed.\n");
    2116              : 
    2117       215267 :   if (!integer_onep (info->assumptions))
    2118              :     {
    2119         7852 :       if (dump_enabled_p ())
    2120              :         {
    2121           84 :           dump_printf_loc (MSG_NOTE, vect_location,
    2122              :                            "Loop to be versioned with assumption ");
    2123           84 :           dump_generic_expr (MSG_NOTE, TDF_SLIM, info->assumptions);
    2124           84 :           dump_printf (MSG_NOTE, "\n");
    2125              :         }
    2126              : 
    2127              :       /* We consider to vectorize this loop by versioning it under
    2128              :          some assumptions.  In order to do this, we need to clear
    2129              :          existing information computed by scev and niter analyzer.  */
    2130         7852 :       scev_reset_htab ();
    2131         7852 :       free_numbers_of_iterations_estimates (loop);
    2132              :       /* Also set flag for this loop so that following scev and niter
    2133              :          analysis are done under the assumptions.  */
    2134         7852 :       loop_constraint_set (loop, LOOP_C_FINITE);
    2135              :     }
    2136              :   else
    2137              :     /* Clear the existing niter information to make sure the nonwrapping flag
    2138              :        will be calculated and set propriately.  */
    2139       207415 :     free_numbers_of_iterations_estimates (loop);
    2140              : 
    2141       215267 :   return opt_result::success ();
    2142              : }
    2143              : 
    2144              : 
    2145              : /* Determine if operating on full vectors for LOOP_VINFO might leave
    2146              :    some scalar iterations still to do.  If so, decide how we should
    2147              :    handle those scalar iterations.  The possibilities are:
    2148              : 
    2149              :    (1) Make LOOP_VINFO operate on partial vectors instead of full vectors.
    2150              :        In this case:
    2151              : 
    2152              :          LOOP_VINFO_USING_PARTIAL_VECTORS_P == true
    2153              :          LOOP_VINFO_PEELING_FOR_NITER == false
    2154              : 
    2155              :    (2) Make LOOP_VINFO operate on full vectors and use an epilogue loop
    2156              :        to handle the remaining scalar iterations.  In this case:
    2157              : 
    2158              :          LOOP_VINFO_USING_PARTIAL_VECTORS_P == false
    2159              :          LOOP_VINFO_PEELING_FOR_NITER == true
    2160              : 
    2161              :    The MASKED_P argument specifies to what extent
    2162              :    param_vect_partial_vector_usage is to be honored.  For MASKED_P == 0
    2163              :    no partial vectors are to be used, for MASKED_P == -1 it's
    2164              :    param_vect_partial_vector_usage that gets to decide whether we may
    2165              :    consider partial vector usage.  For MASKED_P == 1 partial vectors
    2166              :    may be used if possible.
    2167              : 
    2168              :  */
    2169              : 
    2170              : static opt_result
    2171       160574 : vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
    2172              :                                             int masked_p)
    2173              : {
    2174              :   /* Determine whether there would be any scalar iterations left over.  */
    2175       160574 :   bool need_peeling_or_partial_vectors_p
    2176       160574 :     = vect_need_peeling_or_partial_vectors_p (loop_vinfo);
    2177              : 
    2178              :   /* Decide whether to vectorize the loop with partial vectors.  */
    2179       160574 :   LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2180       160574 :   if (masked_p == 0
    2181       160574 :       || (masked_p == -1 && param_vect_partial_vector_usage == 0))
    2182              :     /* If requested explicitly do not use partial vectors.  */
    2183              :     LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2184          227 :   else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2185           71 :            && LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo))
    2186            0 :     LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2187          227 :   else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2188           71 :            && need_peeling_or_partial_vectors_p)
    2189              :     {
    2190              :       /* For partial-vector-usage=1, try to push the handling of partial
    2191              :          vectors to the epilogue, with the main loop continuing to operate
    2192              :          on full vectors.
    2193              : 
    2194              :          If we are unrolling we also do not want to use partial vectors. This
    2195              :          is to avoid the overhead of generating multiple masks and also to
    2196              :          avoid having to execute entire iterations of FALSE masked instructions
    2197              :          when dealing with one or less full iterations.
    2198              : 
    2199              :          ??? We could then end up failing to use partial vectors if we
    2200              :          decide to peel iterations into a prologue, and if the main loop
    2201              :          then ends up processing fewer than VF iterations.  */
    2202           48 :       if ((param_vect_partial_vector_usage == 1
    2203           15 :            || loop_vinfo->suggested_unroll_factor > 1)
    2204           33 :           && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    2205           70 :           && !vect_known_niters_smaller_than_vf (loop_vinfo))
    2206              :         ;
    2207              :       else
    2208           36 :         LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2209              :     }
    2210              : 
    2211       160574 :   if (LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2212            0 :       && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    2213            0 :     return opt_result::failure_at (vect_location,
    2214              :                                    "not vectorized: loop needs but cannot "
    2215              :                                    "use partial vectors\n");
    2216              : 
    2217       160574 :   if (dump_enabled_p ())
    2218        13204 :     dump_printf_loc (MSG_NOTE, vect_location,
    2219              :                      "operating on %s vectors%s.\n",
    2220        13204 :                      LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
    2221              :                      ? "partial" : "full",
    2222        13204 :                      LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    2223              :                      ? " for epilogue loop" : "");
    2224              : 
    2225       160574 :   LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
    2226       321148 :     = (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
    2227       160574 :        && need_peeling_or_partial_vectors_p);
    2228              : 
    2229       160574 :   return opt_result::success ();
    2230              : }
    2231              : 
    2232              : /* Function vect_analyze_loop_2.
    2233              : 
    2234              :    Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
    2235              :    analyses will record information in some members of LOOP_VINFO.  FATAL
    2236              :    indicates if some analysis meets fatal error.  If one non-NULL pointer
    2237              :    SUGGESTED_UNROLL_FACTOR is provided, it's intent to be filled with one
    2238              :    worked out suggested unroll factor, while one NULL pointer shows it's
    2239              :    going to apply the suggested unroll factor.
    2240              :    SINGLE_LANE_SLP_DONE_FOR_SUGGESTED_UF is to hold whether single-lane
    2241              :    slp was forced when the suggested unroll factor was worked out.  */
    2242              : static opt_result
    2243       540724 : vect_analyze_loop_2 (loop_vec_info loop_vinfo, int masked_p, bool &fatal,
    2244              :                      unsigned *suggested_unroll_factor,
    2245              :                      bool& single_lane_slp_done_for_suggested_uf)
    2246              : {
    2247       540724 :   opt_result ok = opt_result::success ();
    2248       540724 :   int res;
    2249       540724 :   unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
    2250       540724 :   loop_vec_info orig_loop_vinfo = NULL;
    2251              : 
    2252              :   /* If we are dealing with an epilogue then orig_loop_vinfo points to the
    2253              :      loop_vec_info of the first vectorized loop.  */
    2254       540724 :   if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
    2255        14495 :     orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
    2256              :   else
    2257              :     orig_loop_vinfo = loop_vinfo;
    2258        14495 :   gcc_assert (orig_loop_vinfo);
    2259              : 
    2260              :   /* We can't mask on niters for uncounted loops due to unknown upper bound.  */
    2261       540724 :   if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
    2262        64845 :     LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2263              : 
    2264              :   /* The first group of checks is independent of the vector size.  */
    2265       540724 :   fatal = true;
    2266              : 
    2267       540724 :   if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)
    2268       540724 :       && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)))
    2269            5 :     return opt_result::failure_at (vect_location,
    2270              :                                    "not vectorized: simd if(0)\n");
    2271              : 
    2272              :   /* Find all data references in the loop (which correspond to vdefs/vuses)
    2273              :      and analyze their evolution in the loop.  */
    2274              : 
    2275       540719 :   loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
    2276              : 
    2277              :   /* Check the data references.  */
    2278       540719 :   loop_vinfo->shared->check_datarefs ();
    2279              : 
    2280              :   /* Analyze the data references and also adjust the minimal
    2281              :      vectorization factor according to the loads and stores.  */
    2282              : 
    2283       540719 :   ok = vect_analyze_data_refs (loop_vinfo, &fatal);
    2284       540719 :   if (!ok)
    2285              :     {
    2286        75849 :       if (dump_enabled_p ())
    2287         1375 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2288              :                          "bad data references.\n");
    2289        75849 :       return ok;
    2290              :     }
    2291              : 
    2292              :   /* Check if we are applying unroll factor now.  */
    2293       464870 :   bool applying_suggested_uf = loop_vinfo->suggested_unroll_factor > 1;
    2294       464870 :   gcc_assert (!applying_suggested_uf || !suggested_unroll_factor);
    2295              : 
    2296              :   /* When single-lane SLP was forced and we are applying suggested unroll
    2297              :      factor, keep that decision here.  */
    2298       929740 :   bool force_single_lane = (applying_suggested_uf
    2299       464870 :                             && single_lane_slp_done_for_suggested_uf);
    2300              : 
    2301              :   /* Classify all cross-iteration scalar data-flow cycles.
    2302              :      Cross-iteration cycles caused by virtual phis are analyzed separately.  */
    2303       464870 :   vect_analyze_scalar_cycles (loop_vinfo);
    2304              : 
    2305       464870 :   vect_pattern_recog (loop_vinfo);
    2306              : 
    2307              :   /* Analyze the access patterns of the data-refs in the loop (consecutive,
    2308              :      complex, etc.). FORNOW: Only handle consecutive access pattern.  */
    2309              : 
    2310       464870 :   ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
    2311       464870 :   if (!ok)
    2312              :     {
    2313         8172 :       if (dump_enabled_p ())
    2314          292 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2315              :                          "bad data access.\n");
    2316         8172 :       return ok;
    2317              :     }
    2318              : 
    2319              :   /* Data-flow analysis to detect stmts that do not need to be vectorized.  */
    2320              : 
    2321       456698 :   ok = vect_mark_stmts_to_be_vectorized (loop_vinfo, &fatal);
    2322       456698 :   if (!ok)
    2323              :     {
    2324        47564 :       if (dump_enabled_p ())
    2325          409 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2326              :                          "unexpected pattern.\n");
    2327        47564 :       return ok;
    2328              :     }
    2329              : 
    2330              :   /* While the rest of the analysis below depends on it in some way.  */
    2331       409134 :   fatal = false;
    2332              : 
    2333              :   /* Analyze data dependences between the data-refs in the loop
    2334              :      and adjust the maximum vectorization factor according to
    2335              :      the dependences.
    2336              :      FORNOW: fail at the first data dependence that we encounter.  */
    2337              : 
    2338       409134 :   ok = vect_analyze_data_ref_dependences (loop_vinfo, &max_vf);
    2339       409134 :   if (!ok)
    2340              :     {
    2341        22492 :       if (dump_enabled_p ())
    2342          563 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2343              :                          "bad data dependence.\n");
    2344        22492 :       return ok;
    2345              :     }
    2346       386642 :   LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
    2347              : 
    2348              :   /* Compute the scalar iteration cost.  */
    2349       386642 :   vect_compute_single_scalar_iteration_cost (loop_vinfo);
    2350              : 
    2351       386642 :   bool saved_can_use_partial_vectors_p
    2352              :     = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
    2353              : 
    2354              :   /* This is the point where we can re-start analysis with single-lane
    2355              :      SLP forced.  */
    2356       537778 : start_over:
    2357              : 
    2358              :   /* Check the SLP opportunities in the loop, analyze and build
    2359              :      SLP trees.  */
    2360      1075556 :   ok = vect_analyze_slp (loop_vinfo, loop_vinfo->stmt_vec_infos.length (),
    2361              :                          force_single_lane);
    2362       537778 :   if (!ok)
    2363        19785 :     return ok;
    2364              : 
    2365              :   /* If there are any SLP instances mark them as pure_slp and compute
    2366              :      the overall vectorization factor.  */
    2367       517993 :   if (!vect_make_slp_decision (loop_vinfo))
    2368        68340 :     return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
    2369              : 
    2370       449653 :   if (dump_enabled_p ())
    2371        20930 :     dump_printf_loc (MSG_NOTE, vect_location, "Loop contains only SLP stmts\n");
    2372              : 
    2373              :   /* Dump the vectorization factor from the SLP decision.  */
    2374       449653 :   if (dump_enabled_p ())
    2375              :     {
    2376        20930 :       dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
    2377        20930 :       dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
    2378        20930 :       dump_printf (MSG_NOTE, "\n");
    2379              :     }
    2380              : 
    2381              :   /* We don't expect to have to roll back to anything other than an empty
    2382              :      set of rgroups.  */
    2383       449653 :   gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
    2384              : 
    2385              :   /* Apply the suggested unrolling factor, this was determined by the backend
    2386              :      during finish_cost the first time we ran the analysis for this
    2387              :      vector mode.  */
    2388       449653 :   if (applying_suggested_uf)
    2389          459 :     LOOP_VINFO_VECT_FACTOR (loop_vinfo) *= loop_vinfo->suggested_unroll_factor;
    2390              : 
    2391              :   /* Now the vectorization factor is final.  */
    2392       449653 :   poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    2393       449653 :   gcc_assert (known_ne (vectorization_factor, 0U));
    2394              : 
    2395              :   /* Optimize the SLP graph with the vectorization factor fixed.  */
    2396       449653 :   vect_optimize_slp (loop_vinfo);
    2397              : 
    2398              :   /* Gather the loads reachable from the SLP graph entries.  */
    2399       449653 :   vect_gather_slp_loads (loop_vinfo);
    2400              : 
    2401       449653 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
    2402              :     {
    2403        15241 :       dump_printf_loc (MSG_NOTE, vect_location,
    2404              :                        "vectorization_factor = ");
    2405        15241 :       dump_dec (MSG_NOTE, vectorization_factor);
    2406        15241 :       dump_printf (MSG_NOTE, ", niters = %wd\n",
    2407        15241 :                    LOOP_VINFO_INT_NITERS (loop_vinfo));
    2408              :     }
    2409              : 
    2410       449653 :   if (max_vf != MAX_VECTORIZATION_FACTOR
    2411       449653 :       && maybe_lt (max_vf, LOOP_VINFO_VECT_FACTOR (loop_vinfo)))
    2412           41 :     return opt_result::failure_at (vect_location, "bad data dependence.\n");
    2413              : 
    2414       449612 :   loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
    2415              : 
    2416              :   /* Analyze the alignment of the data-refs in the loop.  */
    2417       449612 :   vect_analyze_data_refs_alignment (loop_vinfo);
    2418              : 
    2419              :   /* Prune the list of ddrs to be tested at run-time by versioning for alias.
    2420              :      It is important to call pruning after vect_analyze_data_ref_accesses,
    2421              :      since we use grouping information gathered by interleaving analysis.  */
    2422       449612 :   ok = vect_prune_runtime_alias_test_list (loop_vinfo);
    2423       449612 :   if (!ok)
    2424        17786 :     return ok;
    2425              : 
    2426              :   /* Do not invoke vect_enhance_data_refs_alignment for epilogue
    2427              :      vectorization, since we do not want to add extra peeling or
    2428              :      add versioning for alignment.  */
    2429       431826 :   if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
    2430              :     /* This pass will decide on using loop versioning and/or loop peeling in
    2431              :        order to enhance the alignment of data references in the loop.  */
    2432       416353 :     ok = vect_enhance_data_refs_alignment (loop_vinfo);
    2433       431826 :   if (!ok)
    2434            0 :     return ok;
    2435              : 
    2436              :   /* Analyze operations in the SLP instances.  We can't simply
    2437              :      remove unsupported SLP instances as this makes the above
    2438              :      SLP kind detection invalid and might also affect the VF.  */
    2439       431826 :   if (! vect_slp_analyze_operations (loop_vinfo))
    2440              :     {
    2441       271252 :       ok = opt_result::failure_at (vect_location,
    2442              :                                    "unsupported SLP instances\n");
    2443       271252 :       goto again;
    2444              :     }
    2445              : 
    2446              :   /* For now, we don't expect to mix both masking and length approaches for one
    2447              :      loop, disable it if both are recorded.  */
    2448       160574 :   if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2449        24138 :       && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ()
    2450       184706 :       && !LOOP_VINFO_LENS (loop_vinfo).is_empty ())
    2451              :     {
    2452            0 :       if (dump_enabled_p ())
    2453            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2454              :                          "can't vectorize a loop with partial vectors"
    2455              :                          " because we don't expect to mix different"
    2456              :                          " approaches with partial vectors for the"
    2457              :                          " same loop.\n");
    2458            0 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2459              :     }
    2460              : 
    2461              :   /* If we still have the option of using partial vectors,
    2462              :      check whether we can generate the necessary loop controls.  */
    2463       160574 :   if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
    2464              :     {
    2465        24138 :       if (!LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
    2466              :         {
    2467        24132 :           if (!vect_verify_full_masking (loop_vinfo)
    2468        24132 :               && !vect_verify_full_masking_avx512 (loop_vinfo))
    2469         6158 :             LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2470              :         }
    2471              :       else /* !LOOP_VINFO_LENS (loop_vinfo).is_empty () */
    2472            6 :         if (!vect_verify_loop_lens (loop_vinfo))
    2473            6 :           LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2474              :     }
    2475              : 
    2476              :   /* Decide whether this loop_vinfo should use partial vectors or peeling,
    2477              :      assuming that the loop will be used as a main loop.  We will redo
    2478              :      this analysis later if we instead decide to use the loop as an
    2479              :      epilogue loop.  */
    2480       160574 :   ok = vect_determine_partial_vectors_and_peeling (loop_vinfo, masked_p);
    2481       160574 :   if (!ok)
    2482            0 :     return ok;
    2483              : 
    2484              :   /* If we're vectorizing a loop that uses length "controls" and
    2485              :      can iterate more than once, we apply decrementing IV approach
    2486              :      in loop control.  */
    2487       160574 :   if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
    2488           36 :       && LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_len
    2489            0 :       && LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) == 0
    2490       160574 :       && !(LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
    2491            0 :            && known_le (LOOP_VINFO_INT_NITERS (loop_vinfo),
    2492              :                         LOOP_VINFO_VECT_FACTOR (loop_vinfo))))
    2493            0 :     LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = true;
    2494              : 
    2495              :   /* If a loop uses length controls and has a decrementing loop control IV,
    2496              :      we will normally pass that IV through a MIN_EXPR to calcaluate the
    2497              :      basis for the length controls.  E.g. in a loop that processes one
    2498              :      element per scalar iteration, the number of elements would be
    2499              :      MIN_EXPR <N, VF>, where N is the number of scalar iterations left.
    2500              : 
    2501              :      This MIN_EXPR approach allows us to use pointer IVs with an invariant
    2502              :      step, since only the final iteration of the vector loop can have
    2503              :      inactive lanes.
    2504              : 
    2505              :      However, some targets have a dedicated instruction for calculating the
    2506              :      preferred length, given the total number of elements that still need to
    2507              :      be processed.  This is encapsulated in the SELECT_VL internal function.
    2508              : 
    2509              :      If the target supports SELECT_VL, we can use it instead of MIN_EXPR
    2510              :      to determine the basis for the length controls.  However, unlike the
    2511              :      MIN_EXPR calculation, the SELECT_VL calculation can decide to make
    2512              :      lanes inactive in any iteration of the vector loop, not just the last
    2513              :      iteration.  This SELECT_VL approach therefore requires us to use pointer
    2514              :      IVs with variable steps.
    2515              : 
    2516              :      Once we've decided how many elements should be processed by one
    2517              :      iteration of the vector loop, we need to populate the rgroup controls.
    2518              :      If a loop has multiple rgroups, we need to make sure that those rgroups
    2519              :      "line up" (that is, they must be consistent about which elements are
    2520              :      active and which aren't).  This is done by vect_adjust_loop_lens_control.
    2521              : 
    2522              :      In principle, it would be possible to use vect_adjust_loop_lens_control
    2523              :      on either the result of a MIN_EXPR or the result of a SELECT_VL.
    2524              :      However:
    2525              : 
    2526              :      (1) In practice, it only makes sense to use SELECT_VL when a vector
    2527              :          operation will be controlled directly by the result.  It is not
    2528              :          worth using SELECT_VL if it would only be the input to other
    2529              :          calculations.
    2530              : 
    2531              :      (2) If we use SELECT_VL for an rgroup that has N controls, each associated
    2532              :          pointer IV will need N updates by a variable amount (N-1 updates
    2533              :          within the iteration and 1 update to move to the next iteration).
    2534              : 
    2535              :      Because of this, we prefer to use the MIN_EXPR approach whenever there
    2536              :      is more than one length control.
    2537              : 
    2538              :      In addition, SELECT_VL always operates to a granularity of 1 unit.
    2539              :      If we wanted to use it to control an SLP operation on N consecutive
    2540              :      elements, we would need to make the SELECT_VL inputs measure scalar
    2541              :      iterations (rather than elements) and then multiply the SELECT_VL
    2542              :      result by N.  But using SELECT_VL this way is inefficient because
    2543              :      of (1) above.
    2544              : 
    2545              :      2. We don't apply SELECT_VL on single-rgroup when both (1) and (2) are
    2546              :         satisfied:
    2547              : 
    2548              :      (1). LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) is true.
    2549              :      (2). LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant () is true.
    2550              : 
    2551              :      Since SELECT_VL (variable step) will make SCEV analysis failed and then
    2552              :      we will fail to gain benefits of following unroll optimizations. We prefer
    2553              :      using the MIN_EXPR approach in this situation.  */
    2554       160574 :   if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
    2555              :     {
    2556            0 :       tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
    2557            0 :       if (LOOP_VINFO_LENS (loop_vinfo).length () == 1
    2558            0 :           && LOOP_VINFO_LENS (loop_vinfo)[0].factor == 1
    2559            0 :           && (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
    2560              :               || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ()))
    2561            0 :         LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = true;
    2562              : 
    2563            0 :       if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
    2564            0 :         for (auto rgc : LOOP_VINFO_LENS (loop_vinfo))
    2565            0 :           if (rgc.type
    2566            0 :               && !direct_internal_fn_supported_p (IFN_SELECT_VL,
    2567              :                                                   rgc.type, iv_type,
    2568              :                                                   OPTIMIZE_FOR_SPEED))
    2569              :             {
    2570            0 :               LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
    2571            0 :               break;
    2572              :             }
    2573              : 
    2574              :       /* If any of the SLP instances cover more than a single lane
    2575              :          we cannot use .SELECT_VL at the moment, even if the number
    2576              :          of lanes is uniform throughout the SLP graph.  */
    2577            0 :       if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
    2578            0 :         for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
    2579            0 :           if (SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)) != 1
    2580            0 :               && !(SLP_INSTANCE_KIND (inst) == slp_inst_kind_store
    2581            0 :                    && SLP_INSTANCE_TREE (inst)->ldst_lanes))
    2582              :             {
    2583            0 :               LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
    2584            0 :               break;
    2585              :             }
    2586              :     }
    2587              : 
    2588              :   /* If we're vectorizing an epilogue loop, the vectorized loop either needs
    2589              :      to be able to handle fewer than VF scalars, or needs to have a lower VF
    2590              :      than the main loop.  */
    2591       160574 :   if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    2592        14049 :       && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    2593              :     {
    2594        14035 :       poly_uint64 unscaled_vf
    2595        14035 :         = exact_div (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
    2596              :                      orig_loop_vinfo->suggested_unroll_factor);
    2597        14035 :       if (maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo), unscaled_vf))
    2598          396 :         return opt_result::failure_at (vect_location,
    2599              :                                        "Vectorization factor too high for"
    2600              :                                        " epilogue loop.\n");
    2601              :     }
    2602              : 
    2603              :   /* If the epilogue needs peeling for gaps but the main loop doesn't give
    2604              :      up on the epilogue.  */
    2605       160178 :   if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    2606        13653 :       && LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
    2607           73 :       && (LOOP_VINFO_PEELING_FOR_GAPS (orig_loop_vinfo)
    2608              :           != LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
    2609            4 :     return opt_result::failure_at (vect_location,
    2610              :                                    "Epilogue loop requires peeling for gaps "
    2611              :                                    "but main loop does not.\n");
    2612              : 
    2613              :   /* If an epilogue loop is required make sure we can create one.  */
    2614       160174 :   if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
    2615       158868 :       || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
    2616        58436 :       || LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
    2617              :     {
    2618       103325 :       if (dump_enabled_p ())
    2619         6072 :         dump_printf_loc (MSG_NOTE, vect_location, "epilog loop required\n");
    2620       103325 :       if (!vect_can_advance_ivs_p (loop_vinfo)
    2621       206066 :           || !slpeel_can_duplicate_loop_p (loop,
    2622              :                                            LOOP_VINFO_MAIN_EXIT (loop_vinfo),
    2623       102741 :                                            LOOP_VINFO_MAIN_EXIT (loop_vinfo)))
    2624              :         {
    2625          584 :           ok = opt_result::failure_at (vect_location,
    2626              :                                        "not vectorized: can't create required "
    2627              :                                        "epilog loop\n");
    2628          584 :           goto again;
    2629              :         }
    2630              :     }
    2631              : 
    2632              :   /* Check the costings of the loop make vectorizing worthwhile.  */
    2633       159590 :   res = vect_analyze_loop_costing (loop_vinfo, suggested_unroll_factor);
    2634       159590 :   if (res < 0 && !param_vect_allow_possibly_not_worthwhile_vectorizations)
    2635              :     {
    2636        29733 :       ok = opt_result::failure_at (vect_location,
    2637              :                                    "Loop costings may not be worthwhile.\n");
    2638        29733 :       goto again;
    2639              :     }
    2640       129857 :   if (!res)
    2641        28837 :     return opt_result::failure_at (vect_location,
    2642              :                                    "Loop costings not worthwhile.\n");
    2643              : 
    2644              :   /* During peeling, we need to check if number of loop iterations is
    2645              :      enough for both peeled prolog loop and vector loop.  This check
    2646              :      can be merged along with threshold check of loop versioning, so
    2647              :      increase threshold for this case if necessary.
    2648              : 
    2649              :      If we are analyzing an epilogue we still want to check what its
    2650              :      versioning threshold would be.  If we decide to vectorize the epilogues we
    2651              :      will want to use the lowest versioning threshold of all epilogues and main
    2652              :      loop.  This will enable us to enter a vectorized epilogue even when
    2653              :      versioning the loop.  We can't simply check whether the epilogue requires
    2654              :      versioning though since we may have skipped some versioning checks when
    2655              :      analyzing the epilogue.  For instance, checks for alias versioning will be
    2656              :      skipped when dealing with epilogues as we assume we already checked them
    2657              :      for the main loop.  So instead we always check the 'orig_loop_vinfo'.  */
    2658       101020 :   if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo))
    2659              :     {
    2660         9332 :       poly_uint64 niters_th = 0;
    2661         9332 :       unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
    2662              : 
    2663         9332 :       if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
    2664              :         {
    2665              :           /* Niters for peeled prolog loop.  */
    2666         9332 :           if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
    2667              :             {
    2668          115 :               dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
    2669          115 :               tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
    2670          115 :               niters_th += TYPE_VECTOR_SUBPARTS (vectype) - 1;
    2671              :             }
    2672              :           else
    2673         9217 :             niters_th += LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
    2674              :         }
    2675              : 
    2676              :       /* Niters for one vector iteration, excluding a header-only iteration.  */
    2677         9332 :       if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    2678         9328 :         niters_th += LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    2679         9332 :       if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
    2680           48 :         niters_th += 1;
    2681              :       /* One additional iteration because of peeling for gap.  */
    2682         9332 :       if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
    2683           75 :         niters_th += 1;
    2684              : 
    2685              :       /*  Use the same condition as vect_transform_loop to decide when to use
    2686              :           the cost to determine a versioning threshold.  */
    2687         9332 :       if (vect_apply_runtime_profitability_check_p (loop_vinfo)
    2688         9332 :           && ordered_p (th, niters_th))
    2689         6952 :         niters_th = ordered_max (poly_uint64 (th), niters_th);
    2690              : 
    2691         9332 :       LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = niters_th;
    2692              :     }
    2693              : 
    2694       101020 :   gcc_assert (known_eq (vectorization_factor,
    2695              :                         LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
    2696              : 
    2697       101020 :   single_lane_slp_done_for_suggested_uf = force_single_lane;
    2698              : 
    2699              :   /* Ok to vectorize!  */
    2700       101020 :   LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
    2701       101020 :   return opt_result::success ();
    2702              : 
    2703       301569 : again:
    2704              :   /* Ensure that "ok" is false (with an opt_problem if dumping is enabled).  */
    2705       301569 :   gcc_assert (!ok);
    2706              : 
    2707              :   /* Try again with single-lane SLP.  */
    2708       301569 :   if (force_single_lane)
    2709       149451 :     return ok;
    2710              : 
    2711              :   /* If we are applying suggested unroll factor, we don't need to
    2712              :      re-try any more as we want to keep the SLP mode fixed.  */
    2713       152118 :   if (applying_suggested_uf)
    2714           10 :     return ok;
    2715              : 
    2716              :   /* Likewise if the grouped loads or stores in the SLP cannot be handled
    2717              :      via interleaving or lane instructions.  */
    2718              :   slp_instance instance;
    2719              :   slp_tree node;
    2720              :   unsigned i, j;
    2721       409017 :   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
    2722              :     {
    2723       257881 :       if (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance)) != vect_internal_def)
    2724            0 :         continue;
    2725              : 
    2726       257881 :       stmt_vec_info vinfo;
    2727       257881 :       vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0];
    2728       257881 :       if (!vinfo || !STMT_VINFO_GROUPED_ACCESS (vinfo))
    2729       253286 :         continue;
    2730         4595 :       vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
    2731         4595 :       unsigned int size = DR_GROUP_SIZE (vinfo);
    2732         4595 :       tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
    2733         4595 :       if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
    2734         8512 :          && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
    2735         9184 :          && ! vect_grouped_store_supported (vectype, size))
    2736          672 :         return opt_result::failure_at (vinfo->stmt,
    2737              :                                        "unsupported grouped store\n");
    2738       263263 :       FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node)
    2739              :         {
    2740         4156 :           vinfo = SLP_TREE_REPRESENTATIVE (node);
    2741         4156 :           if (STMT_VINFO_GROUPED_ACCESS (vinfo))
    2742              :             {
    2743         2026 :               vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
    2744         2026 :               bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
    2745         2026 :               size = DR_GROUP_SIZE (vinfo);
    2746         2026 :               vectype = SLP_TREE_VECTYPE (node);
    2747         2026 :               if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
    2748         2026 :                   && ! vect_grouped_load_supported (vectype, single_element_p,
    2749              :                                                     size))
    2750          300 :                 return opt_result::failure_at (vinfo->stmt,
    2751              :                                                "unsupported grouped load\n");
    2752              :             }
    2753              :         }
    2754              :     }
    2755              : 
    2756              :   /* Roll back state appropriately.  Force single-lane SLP this time.  */
    2757       151136 :   force_single_lane = true;
    2758       151136 :   if (dump_enabled_p ())
    2759         3994 :     dump_printf_loc (MSG_NOTE, vect_location,
    2760              :                      "re-trying with single-lane SLP\n");
    2761              : 
    2762              :   /* Reset the vectorization factor.  */
    2763       151136 :   LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
    2764              :   /* Free the SLP instances.  */
    2765       408038 :   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
    2766       256902 :     vect_free_slp_instance (instance);
    2767       151136 :   LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
    2768              :   /* Reset altered state on stmts.  */
    2769       723120 :   for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
    2770              :     {
    2771       420848 :       basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
    2772       420848 :       for (gimple_stmt_iterator si = gsi_start_phis (bb);
    2773       760512 :            !gsi_end_p (si); gsi_next (&si))
    2774              :         {
    2775       339664 :           stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
    2776       339664 :           if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
    2777       339664 :               || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
    2778              :             {
    2779              :               /* vectorizable_reduction adjusts reduction stmt def-types,
    2780              :                  restore them to that of the PHI.  */
    2781        27004 :               STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info))
    2782        27004 :                 = STMT_VINFO_DEF_TYPE (stmt_info);
    2783        27004 :               STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
    2784              :                                         (STMT_VINFO_REDUC_DEF (stmt_info)))
    2785        27004 :                 = STMT_VINFO_DEF_TYPE (stmt_info);
    2786              :             }
    2787              :         }
    2788              :     }
    2789              :   /* Free optimized alias test DDRS.  */
    2790       151136 :   LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
    2791       151136 :   LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
    2792       151136 :   LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).release ();
    2793              :   /* Reset target cost data.  */
    2794       151136 :   delete loop_vinfo->vector_costs;
    2795       151136 :   loop_vinfo->vector_costs = nullptr;
    2796              :   /* Reset accumulated rgroup information.  */
    2797       151136 :   LOOP_VINFO_MASKS (loop_vinfo).mask_set.empty ();
    2798       151136 :   release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
    2799       151136 :   release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo));
    2800              :   /* Reset assorted flags.  */
    2801       151136 :   LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo) = false;
    2802       151136 :   LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = false;
    2803       151136 :   LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = 0;
    2804       151136 :   LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = 0;
    2805       151136 :   LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    2806       151136 :     = saved_can_use_partial_vectors_p;
    2807       151136 :   LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2808       151136 :   LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
    2809       151136 :   LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
    2810       151136 :   LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
    2811              : 
    2812       151136 :   if (loop_vinfo->scan_map)
    2813          122 :     loop_vinfo->scan_map->empty ();
    2814              : 
    2815       151136 :   goto start_over;
    2816              : }
    2817              : 
    2818              : /* Return true if vectorizing a loop using NEW_LOOP_VINFO appears
    2819              :    to be better than vectorizing it using OLD_LOOP_VINFO.  Assume that
    2820              :    OLD_LOOP_VINFO is better unless something specifically indicates
    2821              :    otherwise.
    2822              : 
    2823              :    Note that this deliberately isn't a partial order.  */
    2824              : 
    2825              : static bool
    2826        36217 : vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo,
    2827              :                           loop_vec_info old_loop_vinfo)
    2828              : {
    2829        36217 :   struct loop *loop = LOOP_VINFO_LOOP (new_loop_vinfo);
    2830        36217 :   gcc_assert (LOOP_VINFO_LOOP (old_loop_vinfo) == loop);
    2831              : 
    2832        36217 :   poly_int64 new_vf = LOOP_VINFO_VECT_FACTOR (new_loop_vinfo);
    2833        36217 :   poly_int64 old_vf = LOOP_VINFO_VECT_FACTOR (old_loop_vinfo);
    2834              : 
    2835              :   /* Always prefer a VF of loop->simdlen over any other VF.  */
    2836        36217 :   if (loop->simdlen)
    2837              :     {
    2838            0 :       bool new_simdlen_p = known_eq (new_vf, loop->simdlen);
    2839            0 :       bool old_simdlen_p = known_eq (old_vf, loop->simdlen);
    2840            0 :       if (new_simdlen_p != old_simdlen_p)
    2841              :         return new_simdlen_p;
    2842              :     }
    2843              : 
    2844        36217 :   const auto *old_costs = old_loop_vinfo->vector_costs;
    2845        36217 :   const auto *new_costs = new_loop_vinfo->vector_costs;
    2846        36217 :   if (loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo))
    2847         1573 :     return new_costs->better_epilogue_loop_than_p (old_costs, main_loop);
    2848              : 
    2849        34644 :   return new_costs->better_main_loop_than_p (old_costs);
    2850              : }
    2851              : 
    2852              : /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO.  Return
    2853              :    true if we should.  */
    2854              : 
    2855              : static bool
    2856        36217 : vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
    2857              :                         loop_vec_info old_loop_vinfo)
    2858              : {
    2859        36217 :   if (!vect_better_loop_vinfo_p (new_loop_vinfo, old_loop_vinfo))
    2860              :     return false;
    2861              : 
    2862         1487 :   if (dump_enabled_p ())
    2863           18 :     dump_printf_loc (MSG_NOTE, vect_location,
    2864              :                      "***** Preferring vector mode %s to vector mode %s\n",
    2865           18 :                      GET_MODE_NAME (new_loop_vinfo->vector_mode),
    2866           18 :                      GET_MODE_NAME (old_loop_vinfo->vector_mode));
    2867              :   return true;
    2868              : }
    2869              : 
    2870              : /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
    2871              :    not NULL.  When MASKED_P is not -1 override the default
    2872              :    LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
    2873              :    Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
    2874              :    mode useful to analyze.
    2875              :    Return the loop_vinfo on success and wrapped null on failure.  */
    2876              : 
    2877              : static opt_loop_vec_info
    2878       540265 : vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
    2879              :                      const vect_loop_form_info *loop_form_info,
    2880              :                      loop_vec_info orig_loop_vinfo,
    2881              :                      const vector_modes &vector_modes, unsigned &mode_i,
    2882              :                      int masked_p,
    2883              :                      machine_mode &autodetected_vector_mode,
    2884              :                      bool &fatal)
    2885              : {
    2886       540265 :   loop_vec_info loop_vinfo
    2887       540265 :     = vect_create_loop_vinfo (loop, shared, loop_form_info, orig_loop_vinfo);
    2888              : 
    2889       540265 :   machine_mode vector_mode = vector_modes[mode_i];
    2890       540265 :   loop_vinfo->vector_mode = vector_mode;
    2891       540265 :   unsigned int suggested_unroll_factor = 1;
    2892       540265 :   bool single_lane_slp_done_for_suggested_uf = false;
    2893              : 
    2894              :   /* Run the main analysis.  */
    2895       540265 :   opt_result res = vect_analyze_loop_2 (loop_vinfo, masked_p, fatal,
    2896              :                                         &suggested_unroll_factor,
    2897              :                                         single_lane_slp_done_for_suggested_uf);
    2898       540265 :   if (dump_enabled_p ())
    2899        20965 :     dump_printf_loc (MSG_NOTE, vect_location,
    2900              :                      "***** Analysis %s with vector mode %s\n",
    2901        20965 :                      res ? "succeeded" : "failed",
    2902        20965 :                      GET_MODE_NAME (loop_vinfo->vector_mode));
    2903              : 
    2904       540265 :   auto user_unroll = LOOP_VINFO_LOOP (loop_vinfo)->unroll;
    2905       540265 :   if (res && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    2906              :       /* Check to see if the user wants to unroll or if the target wants to.  */
    2907       632366 :       && (suggested_unroll_factor > 1 || user_unroll > 1))
    2908              :     {
    2909          485 :       if (suggested_unroll_factor == 1)
    2910              :         {
    2911           66 :           int assumed_vf = vect_vf_for_cost (loop_vinfo);
    2912           66 :           suggested_unroll_factor = user_unroll / assumed_vf;
    2913           66 :           if (suggested_unroll_factor > 1)
    2914              :             {
    2915           40 :               if (dump_enabled_p ())
    2916           20 :                 dump_printf_loc (MSG_NOTE, vect_location,
    2917              :                          "setting unroll factor to %d based on user requested "
    2918              :                          "unroll factor %d and suggested vectorization "
    2919              :                          "factor: %d\n",
    2920              :                          suggested_unroll_factor, user_unroll, assumed_vf);
    2921              :             }
    2922              :         }
    2923              : 
    2924          485 :         if (suggested_unroll_factor > 1)
    2925              :           {
    2926          459 :             if (dump_enabled_p ())
    2927           62 :               dump_printf_loc (MSG_NOTE, vect_location,
    2928              :                          "***** Re-trying analysis for unrolling"
    2929              :                          " with unroll factor %d and %s slp.\n",
    2930              :                          suggested_unroll_factor,
    2931              :                          single_lane_slp_done_for_suggested_uf
    2932              :                          ? "single-lane" : "");
    2933          459 :             loop_vec_info unroll_vinfo
    2934          459 :                 = vect_create_loop_vinfo (loop, shared, loop_form_info, NULL);
    2935          459 :             unroll_vinfo->vector_mode = vector_mode;
    2936          459 :             unroll_vinfo->suggested_unroll_factor = suggested_unroll_factor;
    2937          459 :             opt_result new_res
    2938          459 :               = vect_analyze_loop_2 (unroll_vinfo, masked_p, fatal, NULL,
    2939              :                                      single_lane_slp_done_for_suggested_uf);
    2940          459 :             if (new_res)
    2941              :               {
    2942          400 :                 delete loop_vinfo;
    2943              :                 loop_vinfo = unroll_vinfo;
    2944              :               }
    2945              :             else
    2946           59 :               delete unroll_vinfo;
    2947              :           }
    2948              : 
    2949              :         /* Record that we have honored a user unroll factor.  */
    2950          485 :         LOOP_VINFO_USER_UNROLL (loop_vinfo) = user_unroll > 1;
    2951              :     }
    2952              : 
    2953              :   /* Remember the autodetected vector mode.  */
    2954       540265 :   if (vector_mode == VOIDmode)
    2955       215267 :     autodetected_vector_mode = loop_vinfo->vector_mode;
    2956              : 
    2957              :   /* Advance mode_i, first skipping modes that would result in the
    2958              :      same analysis result.  */
    2959      2095847 :   while (mode_i + 1 < vector_modes.length ()
    2960      1392530 :          && vect_chooses_same_modes_p (loop_vinfo,
    2961       614739 :                                        vector_modes[mode_i + 1]))
    2962              :     {
    2963       237526 :       if (dump_enabled_p ())
    2964        12670 :         dump_printf_loc (MSG_NOTE, vect_location,
    2965              :                          "***** The result for vector mode %s would"
    2966              :                          " be the same\n",
    2967        12670 :                          GET_MODE_NAME (vector_modes[mode_i + 1]));
    2968       237526 :       mode_i += 1;
    2969              :     }
    2970       540265 :   if (mode_i + 1 < vector_modes.length ()
    2971       917478 :       && vect_chooses_same_modes_p (autodetected_vector_mode,
    2972       377213 :                                     vector_modes[mode_i + 1]))
    2973              :     {
    2974          471 :       if (dump_enabled_p ())
    2975           23 :         dump_printf_loc (MSG_NOTE, vect_location,
    2976              :                          "***** Skipping vector mode %s, which would"
    2977              :                          " repeat the analysis for %s\n",
    2978           23 :                          GET_MODE_NAME (vector_modes[mode_i + 1]),
    2979           23 :                          GET_MODE_NAME (autodetected_vector_mode));
    2980          471 :       mode_i += 1;
    2981              :     }
    2982       540265 :   mode_i++;
    2983              : 
    2984       540265 :   if (!res)
    2985              :     {
    2986       439645 :       delete loop_vinfo;
    2987       439645 :       if (fatal)
    2988        42524 :         gcc_checking_assert (orig_loop_vinfo == NULL);
    2989       439645 :       return opt_loop_vec_info::propagate_failure (res);
    2990              :     }
    2991              : 
    2992       100620 :   return opt_loop_vec_info::success (loop_vinfo);
    2993              : }
    2994              : 
    2995              : /* Function vect_analyze_loop.
    2996              : 
    2997              :    Apply a set of analyses on LOOP, and create a loop_vec_info struct
    2998              :    for it.  The different analyses will record information in the
    2999              :    loop_vec_info struct.  */
    3000              : opt_loop_vec_info
    3001       481919 : vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
    3002              :                    vec_info_shared *shared)
    3003              : {
    3004       481919 :   DUMP_VECT_SCOPE ("analyze_loop_nest");
    3005              : 
    3006       481919 :   if (loop_outer (loop)
    3007       481919 :       && loop_vec_info_for_loop (loop_outer (loop))
    3008       482478 :       && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
    3009          559 :     return opt_loop_vec_info::failure_at (vect_location,
    3010              :                                           "outer-loop already vectorized.\n");
    3011              : 
    3012       481360 :   if (!find_loop_nest (loop, &shared->loop_nest))
    3013        23146 :     return opt_loop_vec_info::failure_at
    3014        23146 :       (vect_location,
    3015              :        "not vectorized: loop nest containing two or more consecutive inner"
    3016              :        " loops cannot be vectorized\n");
    3017              : 
    3018              :   /* Analyze the loop form.  */
    3019       458214 :   vect_loop_form_info loop_form_info;
    3020       458214 :   opt_result res = vect_analyze_loop_form (loop, loop_vectorized_call,
    3021              :                                            &loop_form_info);
    3022       458214 :   if (!res)
    3023              :     {
    3024       179621 :       if (dump_enabled_p ())
    3025         1549 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3026              :                          "bad loop form.\n");
    3027       179621 :       return opt_loop_vec_info::propagate_failure (res);
    3028              :     }
    3029              : 
    3030       278593 :   res = vect_get_datarefs_in_loop (loop, loop_form_info.bbs,
    3031              :                                    &shared->datarefs);
    3032       278593 :   if (!res)
    3033              :     {
    3034        63197 :       if (dump_enabled_p ())
    3035         1639 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3036              :                          "not vectorized: loop contains function "
    3037              :                          "calls or data references that cannot "
    3038              :                          "be analyzed\n");
    3039        63197 :       return opt_loop_vec_info::propagate_failure (res);
    3040              :     }
    3041       215396 :   shared->save_datarefs ();
    3042              : 
    3043       215396 :   res = vect_build_assumptions (loop, shared, &loop_form_info);
    3044       215396 :   if (!res)
    3045          129 :     return opt_loop_vec_info::propagate_failure (res);
    3046              : 
    3047       215267 :   auto_vector_modes vector_modes;
    3048              :   /* Autodetect first vector size we try.  */
    3049       215267 :   vector_modes.safe_push (VOIDmode);
    3050       215267 :   unsigned int autovec_flags
    3051       430534 :     = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
    3052       215267 :                                                     loop->simdlen != 0);
    3053       215267 :   bool pick_lowest_cost_p = ((autovec_flags & VECT_COMPARE_COSTS)
    3054       215267 :                              && !unlimited_cost_model (loop));
    3055       215267 :   machine_mode autodetected_vector_mode = VOIDmode;
    3056       215267 :   opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
    3057       215267 :   unsigned int mode_i = 0;
    3058       215267 :   unsigned HOST_WIDE_INT simdlen = loop->simdlen;
    3059              : 
    3060              :   /* Keep track of the VF for each mode.  Initialize all to 0 which indicates
    3061              :      a mode has not been analyzed.  */
    3062       215267 :   auto_vec<poly_uint64, 8> cached_vf_per_mode;
    3063      2177000 :   for (unsigned i = 0; i < vector_modes.length (); ++i)
    3064       873233 :     cached_vf_per_mode.safe_push (0);
    3065              : 
    3066              :   /* First determine the main loop vectorization mode, either the first
    3067              :      one that works, starting with auto-detecting the vector mode and then
    3068              :      following the targets order of preference, or the one with the
    3069              :      lowest cost if pick_lowest_cost_p.  */
    3070       836273 :   while (1)
    3071              :     {
    3072       525770 :       bool fatal;
    3073       525770 :       unsigned int last_mode_i = mode_i;
    3074              :       /* Set cached VF to -1 prior to analysis, which indicates a mode has
    3075              :          failed.  */
    3076       525770 :       cached_vf_per_mode[last_mode_i] = -1;
    3077       525770 :       opt_loop_vec_info loop_vinfo
    3078       525770 :         = vect_analyze_loop_1 (loop, shared, &loop_form_info,
    3079              :                                NULL, vector_modes, mode_i, -1,
    3080              :                                autodetected_vector_mode, fatal);
    3081       525770 :       if (fatal)
    3082              :         break;
    3083              : 
    3084       483246 :       if (loop_vinfo)
    3085              :         {
    3086              :           /*  Analysis has been successful so update the VF value.  The
    3087              :               VF should always be a multiple of unroll_factor and we want to
    3088              :               capture the original VF here.  */
    3089        92101 :           cached_vf_per_mode[last_mode_i]
    3090        92101 :             = exact_div (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
    3091        92101 :                          loop_vinfo->suggested_unroll_factor);
    3092              :           /* Once we hit the desired simdlen for the first time,
    3093              :              discard any previous attempts.  */
    3094        92101 :           if (simdlen
    3095        92101 :               && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), simdlen))
    3096              :             {
    3097           47 :               delete first_loop_vinfo;
    3098        92101 :               first_loop_vinfo = opt_loop_vec_info::success (NULL);
    3099        92101 :               simdlen = 0;
    3100              :             }
    3101        92054 :           else if (pick_lowest_cost_p
    3102        77738 :                    && first_loop_vinfo
    3103       126698 :                    && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
    3104              :             {
    3105              :               /* Pick loop_vinfo over first_loop_vinfo.  */
    3106         1317 :               delete first_loop_vinfo;
    3107         1317 :               first_loop_vinfo = opt_loop_vec_info::success (NULL);
    3108              :             }
    3109        92101 :           if (first_loop_vinfo == NULL)
    3110              :             first_loop_vinfo = loop_vinfo;
    3111              :           else
    3112              :             {
    3113        33329 :               delete loop_vinfo;
    3114        33329 :               loop_vinfo = opt_loop_vec_info::success (NULL);
    3115              :             }
    3116              : 
    3117              :           /* Commit to first_loop_vinfo if we have no reason to try
    3118              :              alternatives.  */
    3119        92101 :           if (!simdlen && !pick_lowest_cost_p)
    3120              :             break;
    3121              :         }
    3122       468892 :       if (mode_i == vector_modes.length ()
    3123       468892 :           || autodetected_vector_mode == VOIDmode)
    3124              :         break;
    3125              : 
    3126              :       /* Try the next biggest vector size.  */
    3127       310503 :       if (dump_enabled_p ())
    3128         5227 :         dump_printf_loc (MSG_NOTE, vect_location,
    3129              :                          "***** Re-trying analysis with vector mode %s\n",
    3130         5227 :                          GET_MODE_NAME (vector_modes[mode_i]));
    3131       310503 :     }
    3132       215267 :   if (!first_loop_vinfo)
    3133       157817 :     return opt_loop_vec_info::propagate_failure (res);
    3134              : 
    3135        57450 :   if (dump_enabled_p ())
    3136         9935 :     dump_printf_loc (MSG_NOTE, vect_location,
    3137              :                      "***** Choosing vector mode %s\n",
    3138         9935 :                      GET_MODE_NAME (first_loop_vinfo->vector_mode));
    3139              : 
    3140              :   /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
    3141              :      enabled, SIMDUID is not set, it is the innermost loop and we have
    3142              :      either already found the loop's SIMDLEN or there was no SIMDLEN to
    3143              :      begin with.
    3144              :      TODO: Enable epilogue vectorization for loops with SIMDUID set.  */
    3145        57450 :   bool vect_epilogues = (!simdlen
    3146        57448 :                          && loop->inner == NULL
    3147        56887 :                          && param_vect_epilogues_nomask
    3148        55745 :                          && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
    3149              :                            /* No code motion support for multiple epilogues so for now
    3150              :                               not supported when multiple exits.  */
    3151        27206 :                          && !LOOP_VINFO_EARLY_BREAKS (first_loop_vinfo)
    3152        26686 :                          && !loop->simduid
    3153        82719 :                          && loop_cost_model (loop) > VECT_COST_MODEL_VERY_CHEAP);
    3154        57450 :   if (!vect_epilogues)
    3155        43991 :     return first_loop_vinfo;
    3156              : 
    3157              :   /* Now analyze first_loop_vinfo for epilogue vectorization.  */
    3158              : 
    3159              :   /* For epilogues start the analysis from the first mode.  The motivation
    3160              :      behind starting from the beginning comes from cases where the VECTOR_MODES
    3161              :      array may contain length-agnostic and length-specific modes.  Their
    3162              :      ordering is not guaranteed, so we could end up picking a mode for the main
    3163              :      loop that is after the epilogue's optimal mode.  */
    3164        13459 :   int masked_p = -1;
    3165        13459 :   if (!unlimited_cost_model (loop)
    3166        13459 :       && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
    3167              :           != VOIDmode))
    3168              :     {
    3169            5 :       vector_modes[0]
    3170            5 :         = first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
    3171            5 :       cached_vf_per_mode[0] = 0;
    3172              :     }
    3173              :   else
    3174        13454 :     vector_modes[0] = autodetected_vector_mode;
    3175        13459 :   mode_i = 0;
    3176              : 
    3177        13499 :   bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
    3178        13459 :                                    || masked_p == 1);
    3179              :   if (supports_partial_vectors
    3180           40 :       && !partial_vectors_supported_p ()
    3181           40 :       && !LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (first_loop_vinfo))
    3182              :     supports_partial_vectors = false;
    3183        13459 :   poly_uint64 first_vinfo_vf = LOOP_VINFO_VECT_FACTOR (first_loop_vinfo);
    3184              : 
    3185        13459 :   loop_vec_info orig_loop_vinfo = first_loop_vinfo;
    3186        13655 :   do
    3187              :     {
    3188              :       /* Let the user override what the target suggests.  */
    3189        13557 :       if (OPTION_SET_P (param_vect_partial_vector_usage))
    3190           51 :         masked_p = -1;
    3191              : 
    3192        52427 :       while (1)
    3193              :         {
    3194              :           /* If the target does not support partial vectors we can shorten the
    3195              :              number of modes to analyze for the epilogue as we know we can't
    3196              :              pick a mode that would lead to a VF at least as big as the
    3197              :              FIRST_VINFO_VF.  */
    3198        69721 :           if (!supports_partial_vectors
    3199        52427 :               && maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
    3200              :             {
    3201        24672 :               mode_i++;
    3202        49344 :               if (mode_i == vector_modes.length ())
    3203              :                 break;
    3204        30554 :               continue;
    3205              :             }
    3206              :           /* We would need an exhaustive search to find all modes we
    3207              :              skipped but that would lead to the same result as the
    3208              :              analysis it was skipped for and where we'd could check
    3209              :              cached_vf_per_mode against.
    3210              :              Check for the autodetected mode, which is the common
    3211              :              situation on x86 which does not perform cost comparison.  */
    3212        41015 :           if (!supports_partial_vectors
    3213        27717 :               && maybe_ge (cached_vf_per_mode[0], first_vinfo_vf)
    3214        54623 :               && vect_chooses_same_modes_p (autodetected_vector_mode,
    3215        26868 :                                             vector_modes[mode_i]))
    3216              :             {
    3217        13260 :               mode_i++;
    3218        26520 :               if (mode_i == vector_modes.length ())
    3219              :                 break;
    3220        13260 :               continue;
    3221              :             }
    3222              : 
    3223        14495 :           if (dump_enabled_p ())
    3224         3599 :             dump_printf_loc (MSG_NOTE, vect_location,
    3225              :                              "***** Re-trying epilogue analysis with vector "
    3226         3599 :                              "mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
    3227              : 
    3228        14495 :           bool fatal;
    3229        14495 :           opt_loop_vec_info loop_vinfo
    3230        14495 :             = vect_analyze_loop_1 (loop, shared, &loop_form_info,
    3231              :                                    orig_loop_vinfo,
    3232              :                                    vector_modes, mode_i, masked_p,
    3233              :                                    autodetected_vector_mode, fatal);
    3234        14495 :           if (fatal)
    3235              :             break;
    3236              : 
    3237        14495 :           if (loop_vinfo)
    3238              :             {
    3239         8519 :               if (pick_lowest_cost_p
    3240         5466 :                   && orig_loop_vinfo->epilogue_vinfo
    3241        10092 :                   && vect_joust_loop_vinfos (loop_vinfo,
    3242         1573 :                                              orig_loop_vinfo->epilogue_vinfo))
    3243              :                 {
    3244          170 :                   gcc_assert (vect_epilogues);
    3245          170 :                   delete orig_loop_vinfo->epilogue_vinfo;
    3246          170 :                   orig_loop_vinfo->epilogue_vinfo = nullptr;
    3247              :                 }
    3248         8519 :               if (!orig_loop_vinfo->epilogue_vinfo)
    3249         7116 :                 orig_loop_vinfo->epilogue_vinfo = loop_vinfo;
    3250              :               else
    3251              :                 {
    3252         1403 :                   delete loop_vinfo;
    3253         1403 :                   loop_vinfo = opt_loop_vec_info::success (NULL);
    3254              :                 }
    3255              : 
    3256              :               /* For now only allow one epilogue loop, but allow
    3257              :                  pick_lowest_cost_p to replace it, so commit to the
    3258              :                  first epilogue if we have no reason to try alternatives.  */
    3259         8519 :               if (!pick_lowest_cost_p)
    3260              :                 break;
    3261              :             }
    3262              : 
    3263              :           /* Revert back to the default from the suggested preferred
    3264              :              epilogue vectorization mode.  */
    3265        11442 :           masked_p = -1;
    3266        22884 :           if (mode_i == vector_modes.length ())
    3267              :             break;
    3268              :         }
    3269              : 
    3270        13557 :       orig_loop_vinfo = orig_loop_vinfo->epilogue_vinfo;
    3271        13557 :       if (!orig_loop_vinfo)
    3272              :         break;
    3273              : 
    3274              :       /* When we selected a first vectorized epilogue, see if the target
    3275              :          suggests to have another one.  */
    3276         6946 :       masked_p = -1;
    3277         6946 :       if (!unlimited_cost_model (loop)
    3278         3899 :           && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
    3279        10838 :           && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
    3280              :               != VOIDmode))
    3281              :         {
    3282          196 :           vector_modes[0]
    3283           98 :             = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
    3284           98 :           cached_vf_per_mode[0] = 0;
    3285           98 :           mode_i = 0;
    3286              :         }
    3287              :       else
    3288              :         break;
    3289           98 :     }
    3290              :   while (1);
    3291              : 
    3292        13459 :   if (first_loop_vinfo->epilogue_vinfo)
    3293              :     {
    3294         6855 :       poly_uint64 lowest_th
    3295         6855 :         = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
    3296         6855 :       loop_vec_info epilog_vinfo = first_loop_vinfo->epilogue_vinfo;
    3297         6946 :       do
    3298              :         {
    3299         6946 :           poly_uint64 th = LOOP_VINFO_VERSIONING_THRESHOLD (epilog_vinfo);
    3300         6946 :           gcc_assert (!LOOP_REQUIRES_VERSIONING (epilog_vinfo)
    3301              :                       || maybe_ne (lowest_th, 0U));
    3302              :           /* Keep track of the known smallest versioning threshold.  */
    3303         6946 :           if (ordered_p (lowest_th, th))
    3304         6946 :             lowest_th = ordered_min (lowest_th, th);
    3305         6946 :           epilog_vinfo = epilog_vinfo->epilogue_vinfo;
    3306              :         }
    3307         6946 :       while (epilog_vinfo);
    3308         6855 :       LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
    3309         6855 :       if (dump_enabled_p ())
    3310         1554 :         dump_printf_loc (MSG_NOTE, vect_location,
    3311              :                          "***** Choosing epilogue vector mode %s\n",
    3312         1554 :                          GET_MODE_NAME
    3313              :                            (first_loop_vinfo->epilogue_vinfo->vector_mode));
    3314              :     }
    3315              : 
    3316        13459 :   return first_loop_vinfo;
    3317       458214 : }
    3318              : 
    3319              : /* Return true if there is an in-order reduction function for CODE, storing
    3320              :    it in *REDUC_FN if so.  */
    3321              : 
    3322              : static bool
    3323         5531 : fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
    3324              : {
    3325              :   /* We support MINUS_EXPR by negating the operand.  This also preserves an
    3326              :      initial -0.0 since -0.0 - 0.0 (neutral op for MINUS_EXPR) == -0.0 +
    3327              :      (-0.0) = -0.0.  */
    3328         5531 :   if (code == PLUS_EXPR || code == MINUS_EXPR)
    3329              :     {
    3330         4852 :       *reduc_fn = IFN_FOLD_LEFT_PLUS;
    3331            0 :       return true;
    3332              :     }
    3333              :   return false;
    3334              : }
    3335              : 
    3336              : /* Function reduction_fn_for_scalar_code
    3337              : 
    3338              :    Input:
    3339              :    CODE - tree_code of a reduction operations.
    3340              : 
    3341              :    Output:
    3342              :    REDUC_FN - the corresponding internal function to be used to reduce the
    3343              :       vector of partial results into a single scalar result, or IFN_LAST
    3344              :       if the operation is a supported reduction operation, but does not have
    3345              :       such an internal function.
    3346              : 
    3347              :    Return FALSE if CODE currently cannot be vectorized as reduction.  */
    3348              : 
    3349              : bool
    3350      2284022 : reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
    3351              : {
    3352      2284022 :   if (code.is_tree_code ())
    3353      2283964 :     switch (tree_code (code))
    3354              :       {
    3355        14211 :       case MAX_EXPR:
    3356        14211 :         *reduc_fn = IFN_REDUC_MAX;
    3357        14211 :         return true;
    3358              : 
    3359        49298 :       case MIN_EXPR:
    3360        49298 :         *reduc_fn = IFN_REDUC_MIN;
    3361        49298 :         return true;
    3362              : 
    3363      1248079 :       case PLUS_EXPR:
    3364      1248079 :         *reduc_fn = IFN_REDUC_PLUS;
    3365      1248079 :         return true;
    3366              : 
    3367       251006 :       case BIT_AND_EXPR:
    3368       251006 :         *reduc_fn = IFN_REDUC_AND;
    3369       251006 :         return true;
    3370              : 
    3371       270994 :       case BIT_IOR_EXPR:
    3372       270994 :         *reduc_fn = IFN_REDUC_IOR;
    3373       270994 :         return true;
    3374              : 
    3375        45295 :       case BIT_XOR_EXPR:
    3376        45295 :         *reduc_fn = IFN_REDUC_XOR;
    3377        45295 :         return true;
    3378              : 
    3379       405081 :       case MULT_EXPR:
    3380       405081 :       case MINUS_EXPR:
    3381       405081 :         *reduc_fn = IFN_LAST;
    3382       405081 :         return true;
    3383              : 
    3384              :       default:
    3385              :         return false;
    3386              :       }
    3387              :   else
    3388           58 :     switch (combined_fn (code))
    3389              :       {
    3390           34 :       CASE_CFN_FMAX:
    3391           34 :         *reduc_fn = IFN_REDUC_FMAX;
    3392           34 :         return true;
    3393              : 
    3394           24 :       CASE_CFN_FMIN:
    3395           24 :         *reduc_fn = IFN_REDUC_FMIN;
    3396           24 :         return true;
    3397              : 
    3398              :       default:
    3399              :         return false;
    3400              :       }
    3401              : }
    3402              : 
    3403              : /* Set *SBOOL_FN to the corresponding function working on vector masks
    3404              :    for REDUC_FN.  Return true if that exists, false otherwise.  */
    3405              : 
    3406              : static bool
    3407            0 : sbool_reduction_fn_for_fn (internal_fn reduc_fn, internal_fn *sbool_fn)
    3408              : {
    3409            0 :   switch (reduc_fn)
    3410              :     {
    3411            0 :     case IFN_REDUC_AND:
    3412            0 :       *sbool_fn = IFN_REDUC_SBOOL_AND;
    3413            0 :       return true;
    3414            0 :     case IFN_REDUC_IOR:
    3415            0 :       *sbool_fn = IFN_REDUC_SBOOL_IOR;
    3416            0 :       return true;
    3417            0 :     case IFN_REDUC_XOR:
    3418            0 :       *sbool_fn = IFN_REDUC_SBOOL_XOR;
    3419            0 :       return true;
    3420              :     default:
    3421              :       return false;
    3422              :     }
    3423              : }
    3424              : 
    3425              : /* If there is a neutral value X such that a reduction would not be affected
    3426              :    by the introduction of additional X elements, return that X, otherwise
    3427              :    return null.  CODE is the code of the reduction and SCALAR_TYPE is type
    3428              :    of the scalar elements.  If the reduction has just a single initial value
    3429              :    then INITIAL_VALUE is that value, otherwise it is null.
    3430              :    If AS_INITIAL is TRUE the value is supposed to be used as initial value.
    3431              :    In that case no signed zero is returned.  */
    3432              : 
    3433              : tree
    3434        75628 : neutral_op_for_reduction (tree scalar_type, code_helper code,
    3435              :                           tree initial_value, bool as_initial)
    3436              : {
    3437        75628 :   if (code.is_tree_code ())
    3438        75570 :     switch (tree_code (code))
    3439              :       {
    3440        14015 :       case DOT_PROD_EXPR:
    3441        14015 :       case SAD_EXPR:
    3442        14015 :       case MINUS_EXPR:
    3443        14015 :       case BIT_IOR_EXPR:
    3444        14015 :       case BIT_XOR_EXPR:
    3445        14015 :         return build_zero_cst (scalar_type);
    3446        55250 :       case WIDEN_SUM_EXPR:
    3447        55250 :       case PLUS_EXPR:
    3448        55250 :         if (!as_initial && HONOR_SIGNED_ZEROS (scalar_type))
    3449          104 :           return build_real (scalar_type, dconstm0);
    3450              :         else
    3451        55146 :           return build_zero_cst (scalar_type);
    3452              : 
    3453         2252 :       case MULT_EXPR:
    3454         2252 :         return build_one_cst (scalar_type);
    3455              : 
    3456         1573 :       case BIT_AND_EXPR:
    3457         1573 :         return build_all_ones_cst (scalar_type);
    3458              : 
    3459              :       case MAX_EXPR:
    3460              :       case MIN_EXPR:
    3461              :         return initial_value;
    3462              : 
    3463          447 :       default:
    3464          447 :         return NULL_TREE;
    3465              :       }
    3466              :   else
    3467           58 :     switch (combined_fn (code))
    3468              :       {
    3469              :       CASE_CFN_FMIN:
    3470              :       CASE_CFN_FMAX:
    3471              :         return initial_value;
    3472              : 
    3473            0 :       default:
    3474            0 :         return NULL_TREE;
    3475              :       }
    3476              : }
    3477              : 
    3478              : /* Error reporting helper for vect_is_simple_reduction below.  GIMPLE statement
    3479              :    STMT is printed with a message MSG. */
    3480              : 
    3481              : static void
    3482          581 : report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
    3483              : {
    3484          581 :   dump_printf_loc (msg_type, vect_location, "%s%G", msg, stmt);
    3485          581 : }
    3486              : 
    3487              : /* Return true if we need an in-order reduction for operation CODE
    3488              :    on type TYPE.  NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
    3489              :    overflow must wrap.  */
    3490              : 
    3491              : bool
    3492       363249 : needs_fold_left_reduction_p (tree type, code_helper code)
    3493              : {
    3494              :   /* CHECKME: check for !flag_finite_math_only too?  */
    3495       363249 :   if (SCALAR_FLOAT_TYPE_P (type))
    3496              :     {
    3497        97595 :       if (code.is_tree_code ())
    3498        97541 :         switch (tree_code (code))
    3499              :           {
    3500              :           case MIN_EXPR:
    3501              :           case MAX_EXPR:
    3502              :             return false;
    3503              : 
    3504        97004 :           default:
    3505        97004 :             return !flag_associative_math;
    3506              :           }
    3507              :       else
    3508           54 :         switch (combined_fn (code))
    3509              :           {
    3510              :           CASE_CFN_FMIN:
    3511              :           CASE_CFN_FMAX:
    3512              :             return false;
    3513              : 
    3514            2 :           default:
    3515            2 :             return !flag_associative_math;
    3516              :           }
    3517              :     }
    3518              : 
    3519       265654 :   if (INTEGRAL_TYPE_P (type))
    3520       265544 :     return (!code.is_tree_code ()
    3521       265544 :             || !operation_no_trapping_overflow (type, tree_code (code)));
    3522              : 
    3523          110 :   if (SAT_FIXED_POINT_TYPE_P (type))
    3524              :     return true;
    3525              : 
    3526              :   return false;
    3527              : }
    3528              : 
    3529              : /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
    3530              :    has a handled computation expression.  Store the main reduction
    3531              :    operation in *CODE.  */
    3532              : 
    3533              : static bool
    3534       104097 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
    3535              :                       tree loop_arg, code_helper *code,
    3536              :                       vec<std::pair<ssa_op_iter, use_operand_p> > &path,
    3537              :                       bool inner_loop_of_double_reduc)
    3538              : {
    3539       104097 :   auto_bitmap visited;
    3540       104097 :   tree lookfor = PHI_RESULT (phi);
    3541       104097 :   ssa_op_iter curri;
    3542       104097 :   use_operand_p curr = op_iter_init_phiuse (&curri, phi, SSA_OP_USE);
    3543       216589 :   while (USE_FROM_PTR (curr) != loop_arg)
    3544         8395 :     curr = op_iter_next_use (&curri);
    3545       104097 :   curri.i = curri.numops;
    3546       966853 :   do
    3547              :     {
    3548       966853 :       path.safe_push (std::make_pair (curri, curr));
    3549       966853 :       tree use = USE_FROM_PTR (curr);
    3550       966853 :       if (use == lookfor)
    3551              :         break;
    3552       863467 :       gimple *def = SSA_NAME_DEF_STMT (use);
    3553       863467 :       if (gimple_nop_p (def)
    3554       863467 :           || ! flow_bb_inside_loop_p (loop, gimple_bb (def)))
    3555              :         {
    3556       243028 : pop:
    3557       726051 :           do
    3558              :             {
    3559       726051 :               std::pair<ssa_op_iter, use_operand_p> x = path.pop ();
    3560       726051 :               curri = x.first;
    3561       726051 :               curr = x.second;
    3562       794769 :               do
    3563       794769 :                 curr = op_iter_next_use (&curri);
    3564              :               /* Skip already visited or non-SSA operands (from iterating
    3565              :                  over PHI args).  */
    3566              :               while (curr != NULL_USE_OPERAND_P
    3567      1105804 :                      && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
    3568       274402 :                          || ! bitmap_set_bit (visited,
    3569       274402 :                                               SSA_NAME_VERSION
    3570              :                                                 (USE_FROM_PTR (curr)))));
    3571              :             }
    3572      1452102 :           while (curr == NULL_USE_OPERAND_P && ! path.is_empty ());
    3573       243028 :           if (curr == NULL_USE_OPERAND_P)
    3574              :             break;
    3575              :         }
    3576              :       else
    3577              :         {
    3578       726578 :           if (gimple_code (def) == GIMPLE_PHI)
    3579        74109 :             curr = op_iter_init_phiuse (&curri, as_a <gphi *>(def), SSA_OP_USE);
    3580              :           else
    3581       652469 :             curr = op_iter_init_use (&curri, def, SSA_OP_USE);
    3582              :           while (curr != NULL_USE_OPERAND_P
    3583       868298 :                  && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
    3584       756424 :                      || ! bitmap_set_bit (visited,
    3585       756424 :                                           SSA_NAME_VERSION
    3586              :                                             (USE_FROM_PTR (curr)))))
    3587       141720 :             curr = op_iter_next_use (&curri);
    3588       726578 :           if (curr == NULL_USE_OPERAND_P)
    3589       106139 :             goto pop;
    3590              :         }
    3591              :     }
    3592              :   while (1);
    3593       104097 :   if (dump_file && (dump_flags & TDF_DETAILS))
    3594              :     {
    3595         4268 :       dump_printf_loc (MSG_NOTE, loc, "reduction path: ");
    3596         4268 :       unsigned i;
    3597         4268 :       std::pair<ssa_op_iter, use_operand_p> *x;
    3598        14545 :       FOR_EACH_VEC_ELT (path, i, x)
    3599        10277 :         dump_printf (MSG_NOTE, "%T ", USE_FROM_PTR (x->second));
    3600         4268 :       dump_printf (MSG_NOTE, "\n");
    3601              :     }
    3602              : 
    3603              :   /* Check whether the reduction path detected is valid.  */
    3604       104097 :   bool fail = path.length () == 0;
    3605       104097 :   bool neg = false;
    3606       104097 :   int sign = -1;
    3607       104097 :   *code = ERROR_MARK;
    3608       223869 :   for (unsigned i = 1; i < path.length (); ++i)
    3609              :     {
    3610       126831 :       gimple *use_stmt = USE_STMT (path[i].second);
    3611       126831 :       gimple_match_op op;
    3612       126831 :       if (!gimple_extract_op (use_stmt, &op))
    3613              :         {
    3614              :           fail = true;
    3615         7059 :           break;
    3616              :         }
    3617       125915 :       unsigned int opi = op.num_ops;
    3618       125915 :       if (gassign *assign = dyn_cast<gassign *> (use_stmt))
    3619              :         {
    3620              :           /* The following make sure we can compute the operand index
    3621              :              easily plus it mostly disallows chaining via COND_EXPR condition
    3622              :              operands.  */
    3623       194860 :           for (opi = 0; opi < op.num_ops; ++opi)
    3624       193812 :             if (gimple_assign_rhs1_ptr (assign) + opi == path[i].second->use)
    3625              :               break;
    3626              :         }
    3627         6244 :       else if (gcall *call = dyn_cast<gcall *> (use_stmt))
    3628              :         {
    3629        12497 :           for (opi = 0; opi < op.num_ops; ++opi)
    3630        12497 :             if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
    3631              :               break;
    3632              :         }
    3633       125915 :       if (opi == op.num_ops)
    3634              :         {
    3635              :           fail = true;
    3636              :           break;
    3637              :         }
    3638       124867 :       op.code = canonicalize_code (op.code, op.type);
    3639       124867 :       if (op.code == MINUS_EXPR)
    3640              :         {
    3641         5691 :           op.code = PLUS_EXPR;
    3642              :           /* Track whether we negate the reduction value each iteration.  */
    3643         5691 :           if (op.ops[1] == op.ops[opi])
    3644           34 :             neg = ! neg;
    3645              :         }
    3646       119176 :       else if (op.code == IFN_COND_SUB)
    3647              :         {
    3648            9 :           op.code = IFN_COND_ADD;
    3649              :           /* Track whether we negate the reduction value each iteration.  */
    3650            9 :           if (op.ops[2] == op.ops[opi])
    3651            0 :             neg = ! neg;
    3652              :         }
    3653              :       /* For an FMA the reduction code is the PLUS if the addition chain
    3654              :          is the reduction.  */
    3655       119167 :       else if (op.code == IFN_FMA && opi == 2)
    3656           35 :         op.code = PLUS_EXPR;
    3657       124867 :       if (CONVERT_EXPR_CODE_P (op.code)
    3658       124867 :           && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
    3659              :         ;
    3660       119241 :       else if (*code == ERROR_MARK)
    3661              :         {
    3662       101532 :           *code = op.code;
    3663       101532 :           sign = TYPE_SIGN (op.type);
    3664              :         }
    3665        17709 :       else if (op.code != *code)
    3666              :         {
    3667              :           fail = true;
    3668              :           break;
    3669              :         }
    3670        16149 :       else if ((op.code == MIN_EXPR
    3671        15993 :                 || op.code == MAX_EXPR)
    3672        16162 :                && sign != TYPE_SIGN (op.type))
    3673              :         {
    3674              :           fail = true;
    3675              :           break;
    3676              :         }
    3677              :       /* Check there's only a single stmt the op is used on.  For the
    3678              :          not value-changing tail and the last stmt allow out-of-loop uses,
    3679              :          but not when this is the inner loop of a double reduction.
    3680              :          ???  We could relax this and handle arbitrary live stmts by
    3681              :          forcing a scalar epilogue for example.  */
    3682       123304 :       imm_use_iterator imm_iter;
    3683       123304 :       use_operand_p use_p;
    3684       123304 :       gimple *op_use_stmt;
    3685       123304 :       unsigned cnt = 0;
    3686       129511 :       bool cond_fn_p = op.code.is_internal_fn ()
    3687         6207 :         && (conditional_internal_fn_code (internal_fn (op.code))
    3688       123304 :             != ERROR_MARK);
    3689              : 
    3690       295418 :       FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
    3691              :         {
    3692              :           /* In case of a COND_OP (mask, op1, op2, op1) reduction we should
    3693              :              have op1 twice (once as definition, once as else) in the same
    3694              :              operation.  Enforce this.  */
    3695       172114 :           if (cond_fn_p && op_use_stmt == use_stmt)
    3696              :             {
    3697         6121 :               gcall *call = as_a<gcall *> (use_stmt);
    3698         6121 :               unsigned else_pos
    3699         6121 :                 = internal_fn_else_index (internal_fn (op.code));
    3700         6121 :               if (gimple_call_arg (call, else_pos) != op.ops[opi])
    3701              :                 {
    3702              :                   fail = true;
    3703              :                   break;
    3704              :                 }
    3705        30605 :               for (unsigned int j = 0; j < gimple_call_num_args (call); ++j)
    3706              :                 {
    3707        24484 :                   if (j == else_pos)
    3708         6121 :                     continue;
    3709        18363 :                   if (gimple_call_arg (call, j) == op.ops[opi])
    3710         6121 :                     cnt++;
    3711              :                 }
    3712              :             }
    3713       165993 :           else if (!is_gimple_debug (op_use_stmt)
    3714       165993 :                    && ((*code != ERROR_MARK || inner_loop_of_double_reduc)
    3715         2838 :                        || flow_bb_inside_loop_p (loop,
    3716         2838 :                                                  gimple_bb (op_use_stmt))))
    3717       241577 :             FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
    3718       120793 :               cnt++;
    3719       123304 :         }
    3720              : 
    3721       123304 :       if (cnt != 1)
    3722              :         {
    3723              :           fail = true;
    3724              :           break;
    3725              :         }
    3726              :     }
    3727       111878 :   return ! fail && ! neg && *code != ERROR_MARK;
    3728       104097 : }
    3729              : 
    3730              : bool
    3731           21 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
    3732              :                       tree loop_arg, enum tree_code code)
    3733              : {
    3734           21 :   auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
    3735           21 :   code_helper code_;
    3736           21 :   return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path, false)
    3737           21 :           && code_ == code);
    3738           21 : }
    3739              : 
    3740              : 
    3741              : 
    3742              : /* Function vect_is_simple_reduction
    3743              : 
    3744              :    (1) Detect a cross-iteration def-use cycle that represents a simple
    3745              :    reduction computation.  We look for the following pattern:
    3746              : 
    3747              :    loop_header:
    3748              :      a1 = phi < a0, a2 >
    3749              :      a3 = ...
    3750              :      a2 = operation (a3, a1)
    3751              : 
    3752              :    or
    3753              : 
    3754              :    a3 = ...
    3755              :    loop_header:
    3756              :      a1 = phi < a0, a2 >
    3757              :      a2 = operation (a3, a1)
    3758              : 
    3759              :    such that:
    3760              :    1. operation is commutative and associative and it is safe to
    3761              :       change the order of the computation
    3762              :    2. no uses for a2 in the loop (a2 is used out of the loop)
    3763              :    3. no uses of a1 in the loop besides the reduction operation
    3764              :    4. no uses of a1 outside the loop.
    3765              : 
    3766              :    Conditions 1,4 are tested here.
    3767              :    Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
    3768              : 
    3769              :    (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
    3770              :    nested cycles.
    3771              : 
    3772              :    (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
    3773              :    reductions:
    3774              : 
    3775              :      a1 = phi < a0, a2 >
    3776              :      inner loop (def of a3)
    3777              :      a2 = phi < a3 >
    3778              : 
    3779              :    (4) Detect condition expressions, ie:
    3780              :      for (int i = 0; i < N; i++)
    3781              :        if (a[i] < val)
    3782              :         ret_val = a[i];
    3783              : 
    3784              : */
    3785              : 
    3786              : static stmt_vec_info
    3787       170964 : vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
    3788              :                           gphi **double_reduc)
    3789              : {
    3790       170964 :   gphi *phi = as_a <gphi *> (phi_info->stmt);
    3791       170964 :   gimple *phi_use_stmt = NULL;
    3792       170964 :   imm_use_iterator imm_iter;
    3793       170964 :   use_operand_p use_p;
    3794              : 
    3795              :   /* When double_reduc is NULL we are testing the inner loop of a
    3796              :      double reduction.  */
    3797       170964 :   bool inner_loop_of_double_reduc = double_reduc == NULL;
    3798       170964 :   if (double_reduc)
    3799       169835 :     *double_reduc = NULL;
    3800       170964 :   STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION;
    3801              : 
    3802       170964 :   tree phi_name = PHI_RESULT (phi);
    3803              :   /* ???  If there are no uses of the PHI result the inner loop reduction
    3804              :      won't be detected as possibly double-reduction by vectorizable_reduction
    3805              :      because that tries to walk the PHI arg from the preheader edge which
    3806              :      can be constant.  See PR60382.  */
    3807       170964 :   if (has_zero_uses (phi_name))
    3808              :     return NULL;
    3809       170832 :   class loop *loop = (gimple_bb (phi))->loop_father;
    3810       170832 :   unsigned nphi_def_loop_uses = 0;
    3811       476125 :   FOR_EACH_IMM_USE_FAST (use_p, imm_iter, phi_name)
    3812              :     {
    3813       317796 :       gimple *use_stmt = USE_STMT (use_p);
    3814       317796 :       if (is_gimple_debug (use_stmt))
    3815        87173 :         continue;
    3816              : 
    3817       230623 :       if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
    3818              :         {
    3819        12503 :           if (dump_enabled_p ())
    3820           31 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3821              :                              "intermediate value used outside loop.\n");
    3822              : 
    3823        12503 :           return NULL;
    3824              :         }
    3825              : 
    3826              :       /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
    3827              :          op1 twice (once as definition, once as else) in the same operation.
    3828              :          Only count it as one. */
    3829       218120 :       if (use_stmt != phi_use_stmt)
    3830              :         {
    3831       211639 :           nphi_def_loop_uses++;
    3832       211639 :           phi_use_stmt = use_stmt;
    3833              :         }
    3834        12503 :     }
    3835              : 
    3836       158329 :   tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
    3837       158329 :   if (TREE_CODE (latch_def) != SSA_NAME)
    3838              :     {
    3839         1533 :       if (dump_enabled_p ())
    3840           12 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3841              :                          "reduction: not ssa_name: %T\n", latch_def);
    3842              :       return NULL;
    3843              :     }
    3844              : 
    3845       156796 :   stmt_vec_info def_stmt_info = loop_info->lookup_def (latch_def);
    3846       156796 :   if (!def_stmt_info
    3847       156796 :       || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt_info->stmt)))
    3848              :     return NULL;
    3849              : 
    3850       156618 :   bool nested_in_vect_loop
    3851       156618 :     = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info), loop);
    3852       156618 :   unsigned nlatch_def_loop_uses = 0;
    3853       156618 :   auto_vec<gphi *, 3> lcphis;
    3854       618539 :   FOR_EACH_IMM_USE_FAST (use_p, imm_iter, latch_def)
    3855              :     {
    3856       461921 :       gimple *use_stmt = USE_STMT (use_p);
    3857       461921 :       if (is_gimple_debug (use_stmt))
    3858       144143 :         continue;
    3859       317778 :       if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
    3860       199322 :         nlatch_def_loop_uses++;
    3861              :       else
    3862              :         /* We can have more than one loop-closed PHI.  */
    3863       118456 :         lcphis.safe_push (as_a <gphi *> (use_stmt));
    3864       156618 :     }
    3865              : 
    3866              :   /* If we are vectorizing an inner reduction we are executing that
    3867              :      in the original order only in case we are not dealing with a
    3868              :      double reduction.  */
    3869       156618 :   if (nested_in_vect_loop && !inner_loop_of_double_reduc)
    3870              :     {
    3871         2316 :       if (dump_enabled_p ())
    3872          437 :         report_vect_op (MSG_NOTE, def_stmt_info->stmt,
    3873              :                         "detected nested cycle: ");
    3874              :       return def_stmt_info;
    3875              :     }
    3876              : 
    3877              :   /* When the inner loop of a double reduction ends up with more than
    3878              :      one loop-closed PHI we have failed to classify alternate such
    3879              :      PHIs as double reduction, leading to wrong code.  See PR103237.  */
    3880       155419 :   if (inner_loop_of_double_reduc && lcphis.length () != 1)
    3881              :     {
    3882            1 :       if (dump_enabled_p ())
    3883            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3884              :                          "unhandle double reduction\n");
    3885              :       return NULL;
    3886              :     }
    3887              : 
    3888              :   /* If this isn't a nested cycle or if the nested cycle reduction value
    3889              :      is used outside of the inner loop we cannot handle uses of the reduction
    3890              :      value.  */
    3891       154301 :   if (nlatch_def_loop_uses > 1 || nphi_def_loop_uses > 1)
    3892              :     {
    3893        48840 :       if (dump_enabled_p ())
    3894          412 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3895              :                          "reduction used in loop.\n");
    3896              :       return NULL;
    3897              :     }
    3898              : 
    3899              :   /* If DEF_STMT is a phi node itself, we expect it to have a single argument
    3900              :      defined in the inner loop.  */
    3901       105461 :   if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt))
    3902              :     {
    3903         1385 :       tree op1 = PHI_ARG_DEF (def_stmt, 0);
    3904         1385 :       if (gimple_phi_num_args (def_stmt) != 1
    3905         1385 :           || TREE_CODE (op1) != SSA_NAME)
    3906              :         {
    3907           92 :           if (dump_enabled_p ())
    3908            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3909              :                              "unsupported phi node definition.\n");
    3910              : 
    3911              :           return NULL;
    3912              :         }
    3913              : 
    3914              :       /* Verify there is an inner cycle composed of the PHI phi_use_stmt
    3915              :          and the latch definition op1.  */
    3916         1293 :       gimple *def1 = SSA_NAME_DEF_STMT (op1);
    3917         1293 :       if (gimple_bb (def1)
    3918         1293 :           && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
    3919         1293 :           && loop->inner
    3920         1239 :           && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
    3921         1239 :           && (is_gimple_assign (def1) || is_gimple_call (def1))
    3922         1230 :           && is_a <gphi *> (phi_use_stmt)
    3923         1218 :           && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt))
    3924         1218 :           && (op1 == PHI_ARG_DEF_FROM_EDGE (phi_use_stmt,
    3925              :                                             loop_latch_edge (loop->inner)))
    3926       159040 :           && lcphis.length () == 1)
    3927              :         {
    3928         1129 :           if (dump_enabled_p ())
    3929          144 :             report_vect_op (MSG_NOTE, def_stmt,
    3930              :                             "detected double reduction: ");
    3931              : 
    3932         1129 :           *double_reduc = as_a <gphi *> (phi_use_stmt);
    3933         1129 :           return def_stmt_info;
    3934              :         }
    3935              : 
    3936              :       return NULL;
    3937              :     }
    3938              : 
    3939              :   /* Look for the expression computing latch_def from then loop PHI result.  */
    3940       104076 :   auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
    3941       104076 :   code_helper code;
    3942       104076 :   if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
    3943              :                             path, inner_loop_of_double_reduc))
    3944              :     {
    3945        96295 :       STMT_VINFO_REDUC_CODE (phi_info) = code;
    3946        96295 :       if (code == COND_EXPR && !nested_in_vect_loop)
    3947         8512 :         STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION;
    3948              : 
    3949              :       /* Fill in STMT_VINFO_REDUC_IDX.  */
    3950        96295 :       unsigned i;
    3951       310440 :       for (i = path.length () - 1; i >= 1; --i)
    3952              :         {
    3953       117850 :           gimple *stmt = USE_STMT (path[i].second);
    3954       117850 :           stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
    3955       117850 :           gimple_match_op op;
    3956       117850 :           if (!gimple_extract_op (stmt, &op))
    3957            0 :             gcc_unreachable ();
    3958       117850 :           if (gassign *assign = dyn_cast<gassign *> (stmt))
    3959       111630 :             STMT_VINFO_REDUC_IDX (stmt_info)
    3960       111630 :               = path[i].second->use - gimple_assign_rhs1_ptr (assign);
    3961              :           else
    3962              :             {
    3963         6220 :               gcall *call = as_a<gcall *> (stmt);
    3964         6220 :               STMT_VINFO_REDUC_IDX (stmt_info)
    3965         6220 :                 = path[i].second->use - gimple_call_arg_ptr (call, 0);
    3966              :             }
    3967              :         }
    3968        96295 :       if (dump_enabled_p ())
    3969         4250 :         dump_printf_loc (MSG_NOTE, vect_location,
    3970              :                          "reduction: detected reduction\n");
    3971              : 
    3972              :       return def_stmt_info;
    3973              :     }
    3974              : 
    3975         7781 :   if (dump_enabled_p ())
    3976           99 :     dump_printf_loc (MSG_NOTE, vect_location,
    3977              :                      "reduction: unknown pattern\n");
    3978              : 
    3979              :   return NULL;
    3980       260694 : }
    3981              : 
    3982              : /* Estimate the number of peeled epilogue iterations for LOOP_VINFO.
    3983              :    PEEL_ITERS_PROLOGUE is the number of peeled prologue iterations,
    3984              :    or -1 if not known.  */
    3985              : 
    3986              : static int
    3987       517299 : vect_get_peel_iters_epilogue (loop_vec_info loop_vinfo, int peel_iters_prologue)
    3988              : {
    3989       517299 :   int assumed_vf = vect_vf_for_cost (loop_vinfo);
    3990       517299 :   if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) || peel_iters_prologue == -1)
    3991              :     {
    3992       216099 :       if (dump_enabled_p ())
    3993         4345 :         dump_printf_loc (MSG_NOTE, vect_location,
    3994              :                          "cost model: epilogue peel iters set to vf/2 "
    3995              :                          "because loop iterations are unknown .\n");
    3996       216099 :       return assumed_vf / 2;
    3997              :     }
    3998              :   else
    3999              :     {
    4000       301200 :       int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
    4001       301200 :       peel_iters_prologue = MIN (niters, peel_iters_prologue);
    4002       301200 :       int peel_iters_epilogue = (niters - peel_iters_prologue) % assumed_vf;
    4003              :       /* If we need to peel for gaps, but no peeling is required, we have to
    4004              :          peel VF iterations.  */
    4005       301200 :       if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) && !peel_iters_epilogue)
    4006       517299 :         peel_iters_epilogue = assumed_vf;
    4007              :       return peel_iters_epilogue;
    4008              :     }
    4009              : }
    4010              : 
    4011              : /* Calculate cost of peeling the scalar loop PEEL_ITERS_PROLOGUE times for
    4012              :    a prologue and the corresponding times for the epilogue.  */
    4013              : int
    4014       388573 : vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue)
    4015              : {
    4016       388573 :   int retval = 0;
    4017              : 
    4018       388573 :   int peel_iters_epilogue
    4019       388573 :     = vect_get_peel_iters_epilogue (loop_vinfo, peel_iters_prologue);
    4020              : 
    4021       388573 :   if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
    4022              :     {
    4023              :       /* If peeled iterations are known but number of scalar loop
    4024              :          iterations are unknown, count a taken branch per peeled loop.  */
    4025       148087 :       if (peel_iters_prologue > 0)
    4026        90865 :         retval = builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
    4027       148087 :       if (peel_iters_epilogue > 0)
    4028       147974 :         retval += builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
    4029              :     }
    4030              : 
    4031       777146 :   retval += ((peel_iters_prologue + peel_iters_epilogue)
    4032       388573 :              * loop_vinfo->scalar_costs->body_cost ());
    4033       777146 :   retval += (((peel_iters_prologue != 0) + (peel_iters_epilogue != 0))
    4034       388573 :              * loop_vinfo->scalar_costs->outside_cost ());
    4035              : 
    4036       388573 :   return retval;
    4037              : }
    4038              : 
    4039              : /* Return the count for SI in PEEL_ITERS scalar iterations, including the
    4040              :    inner-loop frequency.  */
    4041              : 
    4042              : static int
    4043       317935 : vect_scalar_peeling_stmt_count (loop_vec_info loop_vinfo,
    4044              :                                 const stmt_info_for_cost &si, int peel_iters)
    4045              : {
    4046       317935 :   int count = si.count;
    4047       317935 :   if (si.stmt_info && stmt_in_inner_loop_p (loop_vinfo, si.stmt_info))
    4048          349 :     count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
    4049       317935 :   return count * peel_iters;
    4050              : }
    4051              : 
    4052              : /* Function vect_estimate_min_profitable_iters
    4053              : 
    4054              :    Return the number of iterations required for the vector version of the
    4055              :    loop to be profitable relative to the cost of the scalar version of the
    4056              :    loop.
    4057              : 
    4058              :    *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
    4059              :    of iterations for vectorization.  -1 value means loop vectorization
    4060              :    is not profitable.  This returned value may be used for dynamic
    4061              :    profitability check.
    4062              : 
    4063              :    *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
    4064              :    for static check against estimated number of iterations.  */
    4065              : 
    4066              : static void
    4067       146456 : vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
    4068              :                                     int *ret_min_profitable_niters,
    4069              :                                     int *ret_min_profitable_estimate,
    4070              :                                     unsigned *suggested_unroll_factor)
    4071              : {
    4072       146456 :   int min_profitable_iters;
    4073       146456 :   int min_profitable_estimate;
    4074       146456 :   int peel_iters_prologue;
    4075       146456 :   int peel_iters_epilogue;
    4076       146456 :   unsigned vec_inside_cost = 0;
    4077       146456 :   int vec_outside_cost = 0;
    4078       146456 :   unsigned vec_prologue_cost = 0;
    4079       146456 :   unsigned vec_epilogue_cost = 0;
    4080       146456 :   int scalar_single_iter_cost = 0;
    4081       146456 :   int scalar_outside_cost = 0;
    4082       146456 :   int assumed_vf = vect_vf_for_cost (loop_vinfo);
    4083       146456 :   int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
    4084       146456 :   vector_costs *target_cost_data = loop_vinfo->vector_costs;
    4085              : 
    4086              :   /* Cost model disabled.  */
    4087       146456 :   if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
    4088              :     {
    4089        17420 :       if (dump_enabled_p ())
    4090        11105 :         dump_printf_loc (MSG_NOTE, vect_location, "cost model disabled.\n");
    4091        17420 :       *ret_min_profitable_niters = 0;
    4092        17420 :       *ret_min_profitable_estimate = 0;
    4093        17420 :       return;
    4094              :     }
    4095              : 
    4096              :   /* Requires loop versioning tests to handle misalignment.  */
    4097       129036 :   if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
    4098              :     {
    4099              :       /*  FIXME: Make cost depend on complexity of individual check.  */
    4100           18 :       unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
    4101           18 :       (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
    4102           18 :       if (dump_enabled_p ())
    4103            2 :         dump_printf (MSG_NOTE,
    4104              :                      "cost model: Adding cost of checks for loop "
    4105              :                      "versioning to treat misalignment.\n");
    4106              :     }
    4107              : 
    4108              :   /* Requires loop versioning with alias checks.  */
    4109       129036 :   if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
    4110              :     {
    4111              :       /*  FIXME: Make cost depend on complexity of individual check.  */
    4112         7297 :       unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
    4113         7297 :       (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
    4114         7297 :       len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
    4115            4 :       if (len)
    4116              :         /* Count LEN - 1 ANDs and LEN comparisons.  */
    4117            4 :         (void) add_stmt_cost (target_cost_data, len * 2 - 1,
    4118              :                               scalar_stmt, vect_prologue);
    4119         7297 :       len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
    4120         1275 :       if (len)
    4121              :         {
    4122              :           /* Count LEN - 1 ANDs and LEN comparisons.  */
    4123         1275 :           unsigned int nstmts = len * 2 - 1;
    4124              :           /* +1 for each bias that needs adding.  */
    4125         2550 :           for (unsigned int i = 0; i < len; ++i)
    4126         1275 :             if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
    4127          154 :               nstmts += 1;
    4128         1275 :           (void) add_stmt_cost (target_cost_data, nstmts,
    4129              :                                 scalar_stmt, vect_prologue);
    4130              :         }
    4131         7297 :       if (dump_enabled_p ())
    4132           32 :         dump_printf (MSG_NOTE,
    4133              :                      "cost model: Adding cost of checks for loop "
    4134              :                      "versioning aliasing.\n");
    4135              :     }
    4136              : 
    4137              :   /* Requires loop versioning with niter checks.  */
    4138       129036 :   if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
    4139              :     {
    4140          908 :       tree assum = LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo);
    4141              : 
    4142              :       /* Count all involved statements in the assumption.
    4143              :          ??? This might over-count in case we CSE later.  */
    4144         6180 :       auto count_stmts = [] (tree *t, int *walk, void *nstmts) -> tree
    4145              :         {
    4146         5272 :           if (!EXPR_P (*t) || is_gimple_val (*t))
    4147         2619 :             *walk = 0;
    4148              :           else
    4149         2653 :             (*(int *) nstmts)++;
    4150         5272 :           return NULL_TREE;
    4151              :         };
    4152              : 
    4153          908 :       int nstmts = 0;
    4154          908 :       walk_tree_without_duplicates (&assum, count_stmts, &nstmts);
    4155              : 
    4156          908 :       (void) add_stmt_cost (target_cost_data, nstmts, scalar_stmt,
    4157              :                             NULL, NULL, NULL_TREE, 0, vect_prologue);
    4158          908 :       if (dump_enabled_p ())
    4159            1 :         dump_printf (MSG_NOTE,
    4160              :                      "cost model: Adding cost of checks for loop "
    4161              :                      "versioning niters.\n");
    4162              :     }
    4163              : 
    4164       129036 :   if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
    4165         8141 :     (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
    4166              :                           vect_prologue);
    4167              : 
    4168              :   /* Count statements in scalar loop.  Using this as scalar cost for a single
    4169              :      iteration for now.
    4170              : 
    4171              :      TODO: Add outer loop support.
    4172              : 
    4173              :      TODO: Consider assigning different costs to different scalar
    4174              :      statements.  */
    4175              : 
    4176       129036 :   scalar_single_iter_cost = loop_vinfo->scalar_costs->total_cost ();
    4177              : 
    4178              :   /* Add additional cost for the peeled instructions in prologue and epilogue
    4179              :      loop.  (For fully-masked loops there will be no peeling.)
    4180              : 
    4181              :      FORNOW: If we don't know the value of peel_iters for prologue or epilogue
    4182              :      at compile-time - we assume it's vf/2 (the worst would be vf-1).
    4183              : 
    4184              :      TODO: Build an expression that represents peel_iters for prologue and
    4185              :      epilogue to be used in a run-time test.  */
    4186              : 
    4187       129036 :   bool prologue_need_br_taken_cost = false;
    4188       129036 :   bool prologue_need_br_not_taken_cost = false;
    4189              : 
    4190              :   /* Calculate peel_iters_prologue.  */
    4191       129036 :   if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
    4192              :     peel_iters_prologue = 0;
    4193       129036 :   else if (npeel < 0)
    4194              :     {
    4195          282 :       peel_iters_prologue = assumed_vf / 2;
    4196          282 :       if (dump_enabled_p ())
    4197           11 :         dump_printf (MSG_NOTE, "cost model: "
    4198              :                      "prologue peel iters set to vf/2.\n");
    4199              : 
    4200              :       /* If peeled iterations are unknown, count a taken branch and a not taken
    4201              :          branch per peeled loop.  Even if scalar loop iterations are known,
    4202              :          vector iterations are not known since peeled prologue iterations are
    4203              :          not known.  Hence guards remain the same.  */
    4204              :       prologue_need_br_taken_cost = true;
    4205              :       prologue_need_br_not_taken_cost = true;
    4206              :     }
    4207              :   else
    4208              :     {
    4209       128754 :       peel_iters_prologue = npeel;
    4210       128754 :       if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_prologue > 0)
    4211              :         /* If peeled iterations are known but number of scalar loop
    4212              :            iterations are unknown, count a taken branch per peeled loop.  */
    4213       129036 :         prologue_need_br_taken_cost = true;
    4214              :     }
    4215              : 
    4216       129036 :   bool epilogue_need_br_taken_cost = false;
    4217       129036 :   bool epilogue_need_br_not_taken_cost = false;
    4218              : 
    4219              :   /* Calculate peel_iters_epilogue.  */
    4220       129036 :   if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    4221              :     /* We need to peel exactly one iteration for gaps.  */
    4222           28 :     peel_iters_epilogue = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
    4223       129008 :   else if (npeel < 0)
    4224              :     {
    4225              :       /* If peeling for alignment is unknown, loop bound of main loop
    4226              :          becomes unknown.  */
    4227          282 :       peel_iters_epilogue = assumed_vf / 2;
    4228          282 :       if (dump_enabled_p ())
    4229           11 :         dump_printf (MSG_NOTE, "cost model: "
    4230              :                      "epilogue peel iters set to vf/2 because "
    4231              :                      "peeling for alignment is unknown.\n");
    4232              : 
    4233              :       /* See the same reason above in peel_iters_prologue calculation.  */
    4234              :       epilogue_need_br_taken_cost = true;
    4235              :       epilogue_need_br_not_taken_cost = true;
    4236              :     }
    4237              :   else
    4238              :     {
    4239       128726 :       peel_iters_epilogue = vect_get_peel_iters_epilogue (loop_vinfo, npeel);
    4240       128726 :       if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_epilogue > 0)
    4241              :         /* If peeled iterations are known but number of scalar loop
    4242              :            iterations are unknown, count a taken branch per peeled loop.  */
    4243       129036 :         epilogue_need_br_taken_cost = true;
    4244              :     }
    4245              : 
    4246              :   /* The way we cummulate peeling costs into the vector prologue/epilogue
    4247              :      cost is a bit awkward given we cannot reuse scalar_costs which is
    4248              :      already computed and also because it cannot take into account any
    4249              :      epilogue vectorization we'll carry out in the end.  */
    4250              : 
    4251       129036 :   stmt_info_for_cost *si;
    4252       129036 :   int j;
    4253              :   /* Add costs associated with peel_iters_prologue.  */
    4254       129036 :   if (peel_iters_prologue)
    4255         1083 :     FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
    4256              :       {
    4257          785 :         int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si,
    4258              :                                                     peel_iters_prologue);
    4259          785 :         (void) add_stmt_cost (target_cost_data, count, si->kind,
    4260              :                               si->stmt_info, si->node, si->vectype,
    4261              :                               si->misalign, vect_prologue);
    4262              :       }
    4263              : 
    4264              :   /* Add costs associated with peel_iters_epilogue.  */
    4265       129036 :   if (peel_iters_epilogue)
    4266       396834 :     FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
    4267              :       {
    4268       317150 :         int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si,
    4269              :                                                     peel_iters_epilogue);
    4270       317150 :         (void) add_stmt_cost (target_cost_data, count, si->kind,
    4271              :                               si->stmt_info, si->node, si->vectype,
    4272              :                               si->misalign, vect_epilogue);
    4273              :       }
    4274              : 
    4275              :   /* Add possible cond_branch_taken/cond_branch_not_taken cost.  */
    4276              : 
    4277       129036 :   if (prologue_need_br_taken_cost)
    4278          282 :     (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
    4279              :                           vect_prologue);
    4280              : 
    4281       129036 :   if (prologue_need_br_not_taken_cost)
    4282          282 :     (void) add_stmt_cost (target_cost_data, 1,
    4283              :                           cond_branch_not_taken, vect_prologue);
    4284              : 
    4285       129036 :   if (epilogue_need_br_taken_cost)
    4286        67386 :     (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
    4287              :                           vect_epilogue);
    4288              : 
    4289       129036 :   if (epilogue_need_br_not_taken_cost)
    4290          282 :     (void) add_stmt_cost (target_cost_data, 1,
    4291              :                           cond_branch_not_taken, vect_epilogue);
    4292              : 
    4293              :   /* Take care of special costs for rgroup controls of partial vectors.  */
    4294           28 :   if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
    4295       129064 :       && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
    4296              :           == vect_partial_vectors_avx512))
    4297              :     {
    4298              :       /* Calculate how many masks we need to generate.  */
    4299           28 :       unsigned int num_masks = 0;
    4300           28 :       bool need_saturation = false;
    4301          115 :       for (auto rgm : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
    4302           31 :         if (rgm.type)
    4303              :           {
    4304           28 :             unsigned nvectors = rgm.factor;
    4305           28 :             num_masks += nvectors;
    4306           28 :             if (TYPE_PRECISION (TREE_TYPE (rgm.compare_type))
    4307           28 :                 < TYPE_PRECISION (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo)))
    4308           10 :               need_saturation = true;
    4309              :           }
    4310              : 
    4311              :       /* ???  The target isn't able to identify the costs below as
    4312              :          producing masks so it cannot penaltize cases where we'd run
    4313              :          out of mask registers for example.  */
    4314              : 
    4315              :       /* ???  We are also failing to account for smaller vector masks
    4316              :          we generate by splitting larger masks in vect_get_loop_mask.  */
    4317              : 
    4318              :       /* In the worst case, we need to generate each mask in the prologue
    4319              :          and in the loop body.  We need one splat per group and one
    4320              :          compare per mask.
    4321              : 
    4322              :          Sometimes the prologue mask will fold to a constant,
    4323              :          so the actual prologue cost might be smaller.  However, it's
    4324              :          simpler and safer to use the worst-case cost; if this ends up
    4325              :          being the tie-breaker between vectorizing or not, then it's
    4326              :          probably better not to vectorize.  */
    4327           28 :       (void) add_stmt_cost (target_cost_data,
    4328              :                             num_masks
    4329           28 :                             + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
    4330              :                             vector_stmt, NULL, NULL, NULL_TREE, 0,
    4331              :                             vect_prologue);
    4332           56 :       (void) add_stmt_cost (target_cost_data,
    4333              :                             num_masks
    4334           56 :                             + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
    4335              :                             vector_stmt, NULL, NULL, NULL_TREE, 0, vect_body);
    4336              : 
    4337              :       /* When we need saturation we need it both in the prologue and
    4338              :          the epilogue.  */
    4339           28 :       if (need_saturation)
    4340              :         {
    4341           10 :           (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
    4342              :                                 NULL, NULL, NULL_TREE, 0, vect_prologue);
    4343           10 :           (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
    4344              :                                 NULL, NULL, NULL_TREE, 0, vect_body);
    4345              :         }
    4346              :     }
    4347            0 :   else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
    4348       129008 :            && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
    4349              :                == vect_partial_vectors_while_ult))
    4350              :     {
    4351              :       /* Calculate how many masks we need to generate.  */
    4352              :       unsigned int num_masks = 0;
    4353              :       rgroup_controls *rgm;
    4354              :       unsigned int num_vectors_m1;
    4355            0 :       FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec,
    4356              :                         num_vectors_m1, rgm)
    4357            0 :         if (rgm->type)
    4358            0 :           num_masks += num_vectors_m1 + 1;
    4359            0 :       gcc_assert (num_masks > 0);
    4360              : 
    4361              :       /* In the worst case, we need to generate each mask in the prologue
    4362              :          and in the loop body.  One of the loop body mask instructions
    4363              :          replaces the comparison in the scalar loop, and since we don't
    4364              :          count the scalar comparison against the scalar body, we shouldn't
    4365              :          count that vector instruction against the vector body either.
    4366              : 
    4367              :          Sometimes we can use unpacks instead of generating prologue
    4368              :          masks and sometimes the prologue mask will fold to a constant,
    4369              :          so the actual prologue cost might be smaller.  However, it's
    4370              :          simpler and safer to use the worst-case cost; if this ends up
    4371              :          being the tie-breaker between vectorizing or not, then it's
    4372              :          probably better not to vectorize.  */
    4373            0 :       (void) add_stmt_cost (target_cost_data, num_masks,
    4374              :                             vector_stmt, NULL, NULL, NULL_TREE, 0,
    4375              :                             vect_prologue);
    4376            0 :       (void) add_stmt_cost (target_cost_data, num_masks - 1,
    4377              :                             vector_stmt, NULL, NULL, NULL_TREE, 0,
    4378              :                             vect_body);
    4379              :     }
    4380       129008 :   else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
    4381              :     {
    4382              :       /* Referring to the functions vect_set_loop_condition_partial_vectors
    4383              :          and vect_set_loop_controls_directly, we need to generate each
    4384              :          length in the prologue and in the loop body if required. Although
    4385              :          there are some possible optimizations, we consider the worst case
    4386              :          here.  */
    4387              : 
    4388            0 :       bool niters_known_p = LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo);
    4389            0 :       signed char partial_load_store_bias
    4390              :         = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    4391            0 :       bool need_iterate_p
    4392            0 :         = (!LOOP_VINFO_EPILOGUE_P (loop_vinfo)
    4393            0 :            && !vect_known_niters_smaller_than_vf (loop_vinfo));
    4394              : 
    4395              :       /* Calculate how many statements to be added.  */
    4396            0 :       unsigned int prologue_stmts = 0;
    4397            0 :       unsigned int body_stmts = 0;
    4398              : 
    4399            0 :       rgroup_controls *rgc;
    4400            0 :       unsigned int num_vectors_m1;
    4401            0 :       FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), num_vectors_m1, rgc)
    4402            0 :         if (rgc->type)
    4403              :           {
    4404              :             /* May need one SHIFT for nitems_total computation.  */
    4405            0 :             unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
    4406            0 :             if (nitems != 1 && !niters_known_p)
    4407            0 :               prologue_stmts += 1;
    4408              : 
    4409              :             /* May need one MAX and one MINUS for wrap around.  */
    4410            0 :             if (vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc))
    4411            0 :               prologue_stmts += 2;
    4412              : 
    4413              :             /* Need one MAX and one MINUS for each batch limit excepting for
    4414              :                the 1st one.  */
    4415            0 :             prologue_stmts += num_vectors_m1 * 2;
    4416              : 
    4417            0 :             unsigned int num_vectors = num_vectors_m1 + 1;
    4418              : 
    4419              :             /* Need to set up lengths in prologue, only one MIN required
    4420              :                for each since start index is zero.  */
    4421            0 :             prologue_stmts += num_vectors;
    4422              : 
    4423              :             /* If we have a non-zero partial load bias, we need one PLUS
    4424              :                to adjust the load length.  */
    4425            0 :             if (partial_load_store_bias != 0)
    4426            0 :               body_stmts += 1;
    4427              : 
    4428            0 :             unsigned int length_update_cost = 0;
    4429            0 :             if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
    4430              :               /* For decrement IV style, Each only need a single SELECT_VL
    4431              :                  or MIN since beginning to calculate the number of elements
    4432              :                  need to be processed in current iteration.  */
    4433              :               length_update_cost = 1;
    4434              :             else
    4435              :               /* For increment IV stype, Each may need two MINs and one MINUS to
    4436              :                  update lengths in body for next iteration.  */
    4437            0 :               length_update_cost = 3;
    4438              : 
    4439            0 :             if (need_iterate_p)
    4440            0 :               body_stmts += length_update_cost * num_vectors;
    4441              :           }
    4442              : 
    4443            0 :       (void) add_stmt_cost (target_cost_data, prologue_stmts,
    4444              :                             scalar_stmt, vect_prologue);
    4445            0 :       (void) add_stmt_cost (target_cost_data, body_stmts,
    4446              :                             scalar_stmt, vect_body);
    4447              :     }
    4448              : 
    4449              :   /* FORNOW: The scalar outside cost is incremented in one of the
    4450              :      following ways:
    4451              : 
    4452              :      1. The vectorizer checks for alignment and aliasing and generates
    4453              :      a condition that allows dynamic vectorization.  A cost model
    4454              :      check is ANDED with the versioning condition.  Hence scalar code
    4455              :      path now has the added cost of the versioning check.
    4456              : 
    4457              :        if (cost > th & versioning_check)
    4458              :          jmp to vector code
    4459              : 
    4460              :      Hence run-time scalar is incremented by not-taken branch cost.
    4461              : 
    4462              :      2. The vectorizer then checks if a prologue is required.  If the
    4463              :      cost model check was not done before during versioning, it has to
    4464              :      be done before the prologue check.
    4465              : 
    4466              :        if (cost <= th)
    4467              :          prologue = scalar_iters
    4468              :        if (prologue == 0)
    4469              :          jmp to vector code
    4470              :        else
    4471              :          execute prologue
    4472              :        if (prologue == num_iters)
    4473              :          go to exit
    4474              : 
    4475              :      Hence the run-time scalar cost is incremented by a taken branch,
    4476              :      plus a not-taken branch, plus a taken branch cost.
    4477              : 
    4478              :      3. The vectorizer then checks if an epilogue is required.  If the
    4479              :      cost model check was not done before during prologue check, it
    4480              :      has to be done with the epilogue check.
    4481              : 
    4482              :        if (prologue == 0)
    4483              :          jmp to vector code
    4484              :        else
    4485              :          execute prologue
    4486              :        if (prologue == num_iters)
    4487              :          go to exit
    4488              :        vector code:
    4489              :          if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
    4490              :            jmp to epilogue
    4491              : 
    4492              :      Hence the run-time scalar cost should be incremented by 2 taken
    4493              :      branches.
    4494              : 
    4495              :      TODO: The back end may reorder the BBS's differently and reverse
    4496              :      conditions/branch directions.  Change the estimates below to
    4497              :      something more reasonable.  */
    4498              : 
    4499              :   /* If the number of iterations is known and we do not do versioning, we can
    4500              :      decide whether to vectorize at compile time.  Hence the scalar version
    4501              :      do not carry cost model guard costs.  */
    4502        60763 :   if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
    4503       189799 :       || LOOP_REQUIRES_VERSIONING (loop_vinfo))
    4504              :     {
    4505              :       /* Cost model check occurs at versioning.  */
    4506        69406 :       if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
    4507         8141 :         scalar_outside_cost += vect_get_stmt_cost (cond_branch_not_taken);
    4508              :       else
    4509              :         {
    4510              :           /* Cost model check occurs at prologue generation.  */
    4511        61265 :           if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
    4512          155 :             scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken)
    4513          155 :               + vect_get_stmt_cost (cond_branch_not_taken);
    4514              :           /* Cost model check occurs at epilogue generation.  */
    4515              :           else
    4516        61110 :             scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken);
    4517              :         }
    4518              :     }
    4519              : 
    4520              :   /* Complete the target-specific cost calculations.  */
    4521       129036 :   loop_vinfo->vector_costs->finish_cost (loop_vinfo->scalar_costs);
    4522       129036 :   vec_prologue_cost = loop_vinfo->vector_costs->prologue_cost ();
    4523       129036 :   vec_inside_cost = loop_vinfo->vector_costs->body_cost ();
    4524       129036 :   vec_epilogue_cost = loop_vinfo->vector_costs->epilogue_cost ();
    4525       129036 :   if (suggested_unroll_factor)
    4526       128646 :     *suggested_unroll_factor
    4527       128646 :       = loop_vinfo->vector_costs->suggested_unroll_factor ();
    4528              : 
    4529       128646 :   if (suggested_unroll_factor && *suggested_unroll_factor > 1
    4530          419 :       && LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) != MAX_VECTORIZATION_FACTOR
    4531            0 :       && !known_le (LOOP_VINFO_VECT_FACTOR (loop_vinfo) *
    4532              :                     *suggested_unroll_factor,
    4533              :                     LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo)))
    4534              :     {
    4535            0 :       if (dump_enabled_p ())
    4536            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4537              :                          "can't unroll as unrolled vectorization factor larger"
    4538              :                          " than maximum vectorization factor: "
    4539              :                          HOST_WIDE_INT_PRINT_UNSIGNED "\n",
    4540              :                          LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo));
    4541            0 :       *suggested_unroll_factor = 1;
    4542              :     }
    4543              : 
    4544       129036 :   vec_outside_cost = (int)(vec_prologue_cost + vec_epilogue_cost);
    4545              : 
    4546       129036 :   if (dump_enabled_p ())
    4547              :     {
    4548         1159 :       dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
    4549         1159 :       dump_printf (MSG_NOTE, "  Vector inside of loop cost: %d\n",
    4550              :                    vec_inside_cost);
    4551         1159 :       dump_printf (MSG_NOTE, "  Vector prologue cost: %d\n",
    4552              :                    vec_prologue_cost);
    4553         1159 :       dump_printf (MSG_NOTE, "  Vector epilogue cost: %d\n",
    4554              :                    vec_epilogue_cost);
    4555         1159 :       dump_printf (MSG_NOTE, "  Scalar iteration cost: %d\n",
    4556              :                    scalar_single_iter_cost);
    4557         1159 :       dump_printf (MSG_NOTE, "  Scalar outside cost: %d\n",
    4558              :                    scalar_outside_cost);
    4559         1159 :       dump_printf (MSG_NOTE, "  Vector outside cost: %d\n",
    4560              :                    vec_outside_cost);
    4561         1159 :       dump_printf (MSG_NOTE, "  prologue iterations: %d\n",
    4562              :                    peel_iters_prologue);
    4563         1159 :       dump_printf (MSG_NOTE, "  epilogue iterations: %d\n",
    4564              :                    peel_iters_epilogue);
    4565              :     }
    4566              : 
    4567              :   /* Calculate number of iterations required to make the vector version
    4568              :      profitable, relative to the loop bodies only.  The following condition
    4569              :      must hold true:
    4570              :      SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
    4571              :      where
    4572              :      SIC = scalar iteration cost, VIC = vector iteration cost,
    4573              :      VOC = vector outside cost, VF = vectorization factor,
    4574              :      NPEEL = prologue iterations + epilogue iterations,
    4575              :      SOC = scalar outside cost for run time cost model check.  */
    4576              : 
    4577       129036 :   int saving_per_viter = (scalar_single_iter_cost * assumed_vf
    4578       129036 :                           - vec_inside_cost);
    4579       129036 :   if (saving_per_viter <= 0)
    4580              :     {
    4581        24962 :       if (LOOP_VINFO_LOOP (loop_vinfo)->force_vectorize)
    4582            0 :         warning_at (vect_location.get_location_t (), OPT_Wopenmp_simd,
    4583              :                     "vectorization did not happen for a simd loop");
    4584              : 
    4585        24962 :       if (dump_enabled_p ())
    4586           38 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    4587              :                          "cost model: the vector iteration cost = %d "
    4588              :                          "divided by the scalar iteration cost = %d "
    4589              :                          "is greater or equal to the vectorization factor = %d"
    4590              :                          ".\n",
    4591              :                          vec_inside_cost, scalar_single_iter_cost, assumed_vf);
    4592        24962 :       *ret_min_profitable_niters = -1;
    4593        24962 :       *ret_min_profitable_estimate = -1;
    4594        24962 :       return;
    4595              :     }
    4596              : 
    4597              :   /* ??? The "if" arm is written to handle all cases; see below for what
    4598              :      we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P.  */
    4599       104074 :   if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    4600              :     {
    4601              :       /* Rewriting the condition above in terms of the number of
    4602              :          vector iterations (vniters) rather than the number of
    4603              :          scalar iterations (niters) gives:
    4604              : 
    4605              :          SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
    4606              : 
    4607              :          <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
    4608              : 
    4609              :          For integer N, X and Y when X > 0:
    4610              : 
    4611              :          N * X > Y <==> N >= (Y /[floor] X) + 1.  */
    4612           20 :       int outside_overhead = (vec_outside_cost
    4613           20 :                               - scalar_single_iter_cost * peel_iters_prologue
    4614           20 :                               - scalar_single_iter_cost * peel_iters_epilogue
    4615              :                               - scalar_outside_cost);
    4616              :       /* We're only interested in cases that require at least one
    4617              :          vector iteration.  */
    4618           20 :       int min_vec_niters = 1;
    4619           20 :       if (outside_overhead > 0)
    4620           16 :         min_vec_niters = outside_overhead / saving_per_viter + 1;
    4621              : 
    4622           20 :       if (dump_enabled_p ())
    4623            8 :         dump_printf (MSG_NOTE, "  Minimum number of vector iterations: %d\n",
    4624              :                      min_vec_niters);
    4625              : 
    4626           20 :       if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    4627              :         {
    4628              :           /* Now that we know the minimum number of vector iterations,
    4629              :              find the minimum niters for which the scalar cost is larger:
    4630              : 
    4631              :              SIC * niters > VIC * vniters + VOC - SOC
    4632              : 
    4633              :              We know that the minimum niters is no more than
    4634              :              vniters * VF + NPEEL, but it might be (and often is) less
    4635              :              than that if a partial vector iteration is cheaper than the
    4636              :              equivalent scalar code.  */
    4637           20 :           int threshold = (vec_inside_cost * min_vec_niters
    4638           20 :                            + vec_outside_cost
    4639           20 :                            - scalar_outside_cost);
    4640           20 :           if (threshold <= 0)
    4641              :             min_profitable_iters = 1;
    4642              :           else
    4643           20 :             min_profitable_iters = threshold / scalar_single_iter_cost + 1;
    4644              :         }
    4645              :       else
    4646              :         /* Convert the number of vector iterations into a number of
    4647              :            scalar iterations.  */
    4648            0 :         min_profitable_iters = (min_vec_niters * assumed_vf
    4649            0 :                                 + peel_iters_prologue
    4650              :                                 + peel_iters_epilogue);
    4651              :     }
    4652              :   else
    4653              :     {
    4654       104054 :       min_profitable_iters = ((vec_outside_cost - scalar_outside_cost)
    4655       104054 :                               * assumed_vf
    4656       104054 :                               - vec_inside_cost * peel_iters_prologue
    4657       104054 :                               - vec_inside_cost * peel_iters_epilogue);
    4658       104054 :       if (min_profitable_iters <= 0)
    4659              :         min_profitable_iters = 0;
    4660              :       else
    4661              :         {
    4662        87873 :           min_profitable_iters /= saving_per_viter;
    4663              : 
    4664        87873 :           if ((scalar_single_iter_cost * assumed_vf * min_profitable_iters)
    4665        87873 :               <= (((int) vec_inside_cost * min_profitable_iters)
    4666        87873 :                   + (((int) vec_outside_cost - scalar_outside_cost)
    4667              :                      * assumed_vf)))
    4668        87873 :             min_profitable_iters++;
    4669              :         }
    4670              :     }
    4671              : 
    4672       104074 :   if (dump_enabled_p ())
    4673         1121 :     dump_printf (MSG_NOTE,
    4674              :                  "  Calculated minimum iters for profitability: %d\n",
    4675              :                  min_profitable_iters);
    4676              : 
    4677       104074 :   if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
    4678       104054 :       && min_profitable_iters < (assumed_vf + peel_iters_prologue))
    4679              :     /* We want the vectorized loop to execute at least once.  */
    4680              :     min_profitable_iters = assumed_vf + peel_iters_prologue;
    4681        23765 :   else if (min_profitable_iters < peel_iters_prologue)
    4682              :     /* For LOOP_VINFO_USING_PARTIAL_VECTORS_P, we need to ensure the
    4683              :        vectorized loop executes at least once.  */
    4684              :     min_profitable_iters = peel_iters_prologue;
    4685              : 
    4686       104074 :   if (dump_enabled_p ())
    4687         1121 :     dump_printf_loc (MSG_NOTE, vect_location,
    4688              :                      "  Runtime profitability threshold = %d\n",
    4689              :                      min_profitable_iters);
    4690              : 
    4691       104074 :   *ret_min_profitable_niters = min_profitable_iters;
    4692              : 
    4693              :   /* Calculate number of iterations required to make the vector version
    4694              :      profitable, relative to the loop bodies only.
    4695              : 
    4696              :      Non-vectorized variant is SIC * niters and it must win over vector
    4697              :      variant on the expected loop trip count.  The following condition must hold true:
    4698              :      SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC  */
    4699              : 
    4700       104074 :   if (vec_outside_cost <= 0)
    4701              :     min_profitable_estimate = 0;
    4702              :   /* ??? This "else if" arm is written to handle all cases; see below for
    4703              :      what we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P.  */
    4704        92702 :   else if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    4705              :     {
    4706              :       /* This is a repeat of the code above, but with + SOC rather
    4707              :          than - SOC.  */
    4708           20 :       int outside_overhead = (vec_outside_cost
    4709           20 :                               - scalar_single_iter_cost * peel_iters_prologue
    4710           20 :                               - scalar_single_iter_cost * peel_iters_epilogue
    4711              :                               + scalar_outside_cost);
    4712           20 :       int min_vec_niters = 1;
    4713           20 :       if (outside_overhead > 0)
    4714           20 :         min_vec_niters = outside_overhead / saving_per_viter + 1;
    4715              : 
    4716           20 :       if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
    4717              :         {
    4718           20 :           int threshold = (vec_inside_cost * min_vec_niters
    4719           20 :                            + vec_outside_cost
    4720           20 :                            + scalar_outside_cost);
    4721           20 :           min_profitable_estimate = threshold / scalar_single_iter_cost + 1;
    4722              :         }
    4723              :       else
    4724              :         min_profitable_estimate = (min_vec_niters * assumed_vf
    4725              :                                    + peel_iters_prologue
    4726              :                                    + peel_iters_epilogue);
    4727              :     }
    4728              :   else
    4729              :     {
    4730        92682 :       min_profitable_estimate = ((vec_outside_cost + scalar_outside_cost)
    4731        92682 :                                  * assumed_vf
    4732        92682 :                                  - vec_inside_cost * peel_iters_prologue
    4733        92682 :                                  - vec_inside_cost * peel_iters_epilogue)
    4734        92682 :                                  / ((scalar_single_iter_cost * assumed_vf)
    4735              :                                    - vec_inside_cost);
    4736              :     }
    4737       104074 :   min_profitable_estimate = MAX (min_profitable_estimate, min_profitable_iters);
    4738       104074 :   if (dump_enabled_p ())
    4739         1121 :     dump_printf_loc (MSG_NOTE, vect_location,
    4740              :                      "  Static estimate profitability threshold = %d\n",
    4741              :                      min_profitable_estimate);
    4742              : 
    4743       104074 :   *ret_min_profitable_estimate = min_profitable_estimate;
    4744              : }
    4745              : 
    4746              : /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
    4747              :    vector elements (not bits) for a vector with NELT elements.  */
    4748              : static void
    4749         2104 : calc_vec_perm_mask_for_shift (unsigned int offset, unsigned int nelt,
    4750              :                               vec_perm_builder *sel)
    4751              : {
    4752              :   /* The encoding is a single stepped pattern.  Any wrap-around is handled
    4753              :      by vec_perm_indices.  */
    4754         2104 :   sel->new_vector (nelt, 1, 3);
    4755         8416 :   for (unsigned int i = 0; i < 3; i++)
    4756         6312 :     sel->quick_push (i + offset);
    4757         2104 : }
    4758              : 
    4759              : /* Checks whether the target supports whole-vector shifts for vectors of mode
    4760              :    MODE.  This is the case if _either_ the platform handles vec_shr_optab, _or_
    4761              :    it supports vec_perm_const with masks for all necessary shift amounts.  */
    4762              : static bool
    4763        13398 : have_whole_vector_shift (machine_mode mode)
    4764              : {
    4765        13398 :   if (can_implement_p (vec_shr_optab, mode))
    4766              :     return true;
    4767              : 
    4768              :   /* Variable-length vectors should be handled via the optab.  */
    4769           63 :   unsigned int nelt;
    4770          126 :   if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    4771              :     return false;
    4772              : 
    4773           63 :   vec_perm_builder sel;
    4774           63 :   vec_perm_indices indices;
    4775          315 :   for (unsigned int i = nelt / 2; i >= 1; i /= 2)
    4776              :     {
    4777          252 :       calc_vec_perm_mask_for_shift (i, nelt, &sel);
    4778          252 :       indices.new_vector (sel, 2, nelt);
    4779          252 :       if (!can_vec_perm_const_p (mode, mode, indices, false))
    4780              :         return false;
    4781              :     }
    4782              :   return true;
    4783           63 : }
    4784              : 
    4785              : /* Return true if (a) STMT_INFO is a DOT_PROD_EXPR reduction whose
    4786              :    multiplication operands have differing signs and (b) we intend
    4787              :    to emulate the operation using a series of signed DOT_PROD_EXPRs.
    4788              :    See vect_emulate_mixed_dot_prod for the actual sequence used.  */
    4789              : 
    4790              : static bool
    4791         2455 : vect_is_emulated_mixed_dot_prod (slp_tree slp_node)
    4792              : {
    4793         2455 :   stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_node);
    4794         2455 :   gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
    4795         2000 :   if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
    4796              :     return false;
    4797              : 
    4798          856 :   tree rhs1 = gimple_assign_rhs1 (assign);
    4799          856 :   tree rhs2 = gimple_assign_rhs2 (assign);
    4800          856 :   if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
    4801              :     return false;
    4802              : 
    4803          645 :   return !directly_supported_p (DOT_PROD_EXPR,
    4804              :                                 SLP_TREE_VECTYPE (slp_node),
    4805          215 :                                 SLP_TREE_VECTYPE
    4806              :                                   (SLP_TREE_CHILDREN (slp_node)[0]),
    4807          215 :                                 optab_vector_mixed_sign);
    4808              : }
    4809              : 
    4810              : /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
    4811              :    functions. Design better to avoid maintenance issues.  */
    4812              : 
    4813              : /* Function vect_model_reduction_cost.
    4814              : 
    4815              :    Models cost for a reduction operation, including the vector ops
    4816              :    generated within the strip-mine loop in some cases, the initial
    4817              :    definition before the loop, and the epilogue code that must be generated.  */
    4818              : 
    4819              : static void
    4820        70007 : vect_model_reduction_cost (loop_vec_info loop_vinfo,
    4821              :                            slp_tree node, internal_fn reduc_fn,
    4822              :                            vect_reduction_type reduction_type,
    4823              :                            int ncopies, stmt_vector_for_cost *cost_vec)
    4824              : {
    4825        70007 :   int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
    4826        70007 :   tree vectype;
    4827        70007 :   machine_mode mode;
    4828        70007 :   class loop *loop = NULL;
    4829              : 
    4830        70007 :   if (loop_vinfo)
    4831        70007 :     loop = LOOP_VINFO_LOOP (loop_vinfo);
    4832              : 
    4833              :   /* Condition reductions generate two reductions in the loop.  */
    4834        70007 :   if (reduction_type == COND_REDUCTION)
    4835          349 :     ncopies *= 2;
    4836              : 
    4837        70007 :   vectype = SLP_TREE_VECTYPE (node);
    4838        70007 :   mode = TYPE_MODE (vectype);
    4839        70007 :   stmt_vec_info orig_stmt_info
    4840        70007 :     = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
    4841              : 
    4842        70007 :   gimple_match_op op;
    4843        70007 :   if (!gimple_extract_op (orig_stmt_info->stmt, &op))
    4844            0 :     gcc_unreachable ();
    4845              : 
    4846        70007 :   if (reduction_type == EXTRACT_LAST_REDUCTION)
    4847              :     /* No extra instructions are needed in the prologue.  The loop body
    4848              :        operations are costed in vectorizable_condition.  */
    4849              :     inside_cost = 0;
    4850        70007 :   else if (reduction_type == FOLD_LEFT_REDUCTION)
    4851              :     {
    4852              :       /* No extra instructions needed in the prologue.  */
    4853         4714 :       prologue_cost = 0;
    4854              : 
    4855         4714 :       if (reduc_fn != IFN_LAST)
    4856              :         /* Count one reduction-like operation per vector.  */
    4857            0 :         inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
    4858              :                                         node, 0, vect_body);
    4859              :       else
    4860              :         {
    4861              :           /* Use NCOPIES deconstructs and NELEMENTS scalar ops.  */
    4862         4714 :           unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
    4863         4714 :           inside_cost = record_stmt_cost (cost_vec, ncopies,
    4864              :                                           vec_deconstruct, node, 0,
    4865              :                                           vect_body);
    4866         4714 :           inside_cost += record_stmt_cost (cost_vec, nelements,
    4867              :                                            scalar_stmt, node, 0,
    4868              :                                            vect_body);
    4869              :         }
    4870              :     }
    4871              :   else
    4872              :     {
    4873              :       /* Add in the cost of the initial definitions.  */
    4874        65293 :       int prologue_stmts;
    4875        65293 :       if (reduction_type == COND_REDUCTION)
    4876              :         /* For cond reductions we have four vectors: initial index, step,
    4877              :            initial result of the data reduction, initial value of the index
    4878              :            reduction.  */
    4879              :         prologue_stmts = 4;
    4880              :       else
    4881              :         /* We need the initial reduction value.  */
    4882        64944 :         prologue_stmts = 1;
    4883        65293 :       prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
    4884              :                                          scalar_to_vec, node, 0,
    4885              :                                          vect_prologue);
    4886              :     }
    4887              : 
    4888              :   /* Determine cost of epilogue code.
    4889              : 
    4890              :      We have a reduction operator that will reduce the vector in one statement.
    4891              :      Also requires scalar extract.  */
    4892              : 
    4893        70007 :   if (!loop || !nested_in_vect_loop_p (loop, orig_stmt_info))
    4894              :     {
    4895        69831 :       if (reduc_fn != IFN_LAST)
    4896              :         {
    4897        49927 :           if (reduction_type == COND_REDUCTION)
    4898              :             {
    4899              :               /* An EQ stmt and an COND_EXPR stmt.  */
    4900           18 :               epilogue_cost += record_stmt_cost (cost_vec, 2,
    4901              :                                                  vector_stmt, node, 0,
    4902              :                                                  vect_epilogue);
    4903              :               /* Reduction of the max index and a reduction of the found
    4904              :                  values.  */
    4905           18 :               epilogue_cost += record_stmt_cost (cost_vec, 2,
    4906              :                                                  vec_to_scalar, node, 0,
    4907              :                                                  vect_epilogue);
    4908              :               /* A broadcast of the max value.  */
    4909           18 :               epilogue_cost += record_stmt_cost (cost_vec, 1,
    4910              :                                                  scalar_to_vec, node, 0,
    4911              :                                                  vect_epilogue);
    4912              :             }
    4913              :           else
    4914              :             {
    4915        49909 :               epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
    4916              :                                                  node, 0, vect_epilogue);
    4917        49909 :               epilogue_cost += record_stmt_cost (cost_vec, 1,
    4918              :                                                  vec_to_scalar, node, 0,
    4919              :                                                  vect_epilogue);
    4920              :             }
    4921              :         }
    4922        19904 :       else if (reduction_type == COND_REDUCTION)
    4923              :         {
    4924          331 :           unsigned estimated_nunits = vect_nunits_for_cost (vectype);
    4925              :           /* Extraction of scalar elements.  */
    4926          331 :           epilogue_cost += record_stmt_cost (cost_vec, 2,
    4927              :                                              vec_deconstruct, node, 0,
    4928              :                                              vect_epilogue);
    4929              :           /* Scalar max reductions via COND_EXPR / MAX_EXPR.  */
    4930          331 :           epilogue_cost += record_stmt_cost (cost_vec,
    4931          331 :                                              2 * estimated_nunits - 3,
    4932              :                                              scalar_stmt, node, 0,
    4933              :                                              vect_epilogue);
    4934              :         }
    4935        19573 :       else if (reduction_type == EXTRACT_LAST_REDUCTION
    4936        19573 :                || reduction_type == FOLD_LEFT_REDUCTION)
    4937              :         /* No extra instructions need in the epilogue.  */
    4938              :         ;
    4939              :       else
    4940              :         {
    4941        14859 :           int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
    4942        14859 :           tree bitsize = TYPE_SIZE (op.type);
    4943        14859 :           int element_bitsize = tree_to_uhwi (bitsize);
    4944        14859 :           int nelements = vec_size_in_bits / element_bitsize;
    4945              : 
    4946        14859 :           if (op.code == COND_EXPR)
    4947           31 :             op.code = MAX_EXPR;
    4948              : 
    4949              :           /* We have a whole vector shift available.  */
    4950         2519 :           if (VECTOR_MODE_P (mode)
    4951        14859 :               && directly_supported_p (op.code, vectype)
    4952        26440 :               && have_whole_vector_shift (mode))
    4953              :             {
    4954              :               /* Final reduction via vector shifts and the reduction operator.
    4955              :                  Also requires scalar extract.  */
    4956        34743 :               epilogue_cost += record_stmt_cost (cost_vec,
    4957        23162 :                                                  exact_log2 (nelements) * 2,
    4958              :                                                  vector_stmt, node, 0,
    4959              :                                                  vect_epilogue);
    4960        11581 :               epilogue_cost += record_stmt_cost (cost_vec, 1,
    4961              :                                                  vec_to_scalar, node, 0,
    4962              :                                                  vect_epilogue);
    4963              :             }
    4964              :           else
    4965              :             /* Use extracts and reduction op for final reduction.  For N
    4966              :                elements, we have N extracts and N-1 reduction ops.  */
    4967         3278 :             epilogue_cost += record_stmt_cost (cost_vec,
    4968         3278 :                                                nelements + nelements - 1,
    4969              :                                                vector_stmt, node, 0,
    4970              :                                                vect_epilogue);
    4971              :         }
    4972              :     }
    4973              : 
    4974        70007 :   if (dump_enabled_p ())
    4975         3100 :     dump_printf (MSG_NOTE,
    4976              :                  "vect_model_reduction_cost: inside_cost = %d, "
    4977              :                  "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost,
    4978              :                  prologue_cost, epilogue_cost);
    4979        70007 : }
    4980              : 
    4981              : /* SEQ is a sequence of instructions that initialize the reduction
    4982              :    described by REDUC_INFO.  Emit them in the appropriate place.  */
    4983              : 
    4984              : static void
    4985          461 : vect_emit_reduction_init_stmts (loop_vec_info loop_vinfo,
    4986              :                                 vect_reduc_info reduc_info, gimple *seq)
    4987              : {
    4988          461 :   if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
    4989              :     {
    4990              :       /* When reusing an accumulator from the main loop, we only need
    4991              :          initialization instructions if the main loop can be skipped.
    4992              :          In that case, emit the initialization instructions at the end
    4993              :          of the guard block that does the skip.  */
    4994           18 :       edge skip_edge = loop_vinfo->skip_main_loop_edge;
    4995           18 :       gcc_assert (skip_edge);
    4996           18 :       gimple_stmt_iterator gsi = gsi_last_bb (skip_edge->src);
    4997           18 :       gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
    4998              :     }
    4999              :   else
    5000              :     {
    5001              :       /* The normal case: emit the initialization instructions on the
    5002              :          preheader edge.  */
    5003          443 :       class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    5004          443 :       gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), seq);
    5005              :     }
    5006          461 : }
    5007              : 
    5008              : /* Get at the initial defs for the reduction PHIs for REDUC_INFO,
    5009              :    which performs a reduction involving GROUP_SIZE scalar statements.
    5010              :    NUMBER_OF_VECTORS is the number of vector defs to create.  If NEUTRAL_OP
    5011              :    is nonnull, introducing extra elements of that value will not change the
    5012              :    result.  */
    5013              : 
    5014              : static void
    5015        21821 : get_initial_defs_for_reduction (loop_vec_info loop_vinfo,
    5016              :                                 vect_reduc_info reduc_info,
    5017              :                                 tree vector_type,
    5018              :                                 vec<tree> *vec_oprnds,
    5019              :                                 unsigned int number_of_vectors,
    5020              :                                 unsigned int group_size, tree neutral_op)
    5021              : {
    5022        21821 :   vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
    5023        21821 :   unsigned HOST_WIDE_INT nunits;
    5024        21821 :   unsigned j, number_of_places_left_in_vector;
    5025        21821 :   unsigned int i;
    5026              : 
    5027        43642 :   gcc_assert (group_size == initial_values.length () || neutral_op);
    5028              : 
    5029              :   /* NUMBER_OF_COPIES is the number of times we need to use the same values in
    5030              :      created vectors. It is greater than 1 if unrolling is performed.
    5031              : 
    5032              :      For example, we have two scalar operands, s1 and s2 (e.g., group of
    5033              :      strided accesses of size two), while NUNITS is four (i.e., four scalars
    5034              :      of this type can be packed in a vector).  The output vector will contain
    5035              :      two copies of each scalar operand: {s1, s2, s1, s2}.  (NUMBER_OF_COPIES
    5036              :      will be 2).
    5037              : 
    5038              :      If GROUP_SIZE > NUNITS, the scalars will be split into several
    5039              :      vectors containing the operands.
    5040              : 
    5041              :      For example, NUNITS is four as before, and the group size is 8
    5042              :      (s1, s2, ..., s8).  We will create two vectors {s1, s2, s3, s4} and
    5043              :      {s5, s6, s7, s8}.  */
    5044              : 
    5045        21821 :   if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
    5046              :     nunits = group_size;
    5047              : 
    5048        21821 :   tree vector_elt_type = TREE_TYPE (vector_type);
    5049        21821 :   number_of_places_left_in_vector = nunits;
    5050        21821 :   bool constant_p = true;
    5051        21821 :   tree_vector_builder elts (vector_type, nunits, 1);
    5052        21821 :   elts.quick_grow (nunits);
    5053        21821 :   gimple_seq ctor_seq = NULL;
    5054        21821 :   if (neutral_op
    5055        43048 :       && !useless_type_conversion_p (vector_elt_type,
    5056        21227 :                                      TREE_TYPE (neutral_op)))
    5057              :     {
    5058          248 :       if (VECTOR_BOOLEAN_TYPE_P (vector_type))
    5059          227 :         neutral_op = gimple_build (&ctor_seq, COND_EXPR,
    5060              :                                    vector_elt_type,
    5061              :                                    neutral_op,
    5062              :                                    build_all_ones_cst (vector_elt_type),
    5063              :                                    build_zero_cst (vector_elt_type));
    5064              :       else
    5065           21 :         neutral_op = gimple_convert (&ctor_seq, vector_elt_type, neutral_op);
    5066              :     }
    5067       204605 :   for (j = 0; j < nunits * number_of_vectors; ++j)
    5068              :     {
    5069       182784 :       tree op;
    5070       182784 :       i = j % group_size;
    5071              : 
    5072              :       /* Get the def before the loop.  In reduction chain we have only
    5073              :          one initial value.  Else we have as many as PHIs in the group.  */
    5074       182784 :       if (i >= initial_values.length () || (j > i && neutral_op))
    5075              :         op = neutral_op;
    5076              :       else
    5077              :         {
    5078        51550 :           if (!useless_type_conversion_p (vector_elt_type,
    5079        25775 :                                           TREE_TYPE (initial_values[i])))
    5080              :             {
    5081          263 :               if (VECTOR_BOOLEAN_TYPE_P (vector_type))
    5082          478 :                 initial_values[i] = gimple_build (&ctor_seq, COND_EXPR,
    5083              :                                                   vector_elt_type,
    5084          239 :                                                   initial_values[i],
    5085              :                                                   build_all_ones_cst
    5086              :                                                     (vector_elt_type),
    5087              :                                                   build_zero_cst
    5088              :                                                     (vector_elt_type));
    5089              :               else
    5090           48 :                 initial_values[i] = gimple_convert (&ctor_seq,
    5091              :                                                     vector_elt_type,
    5092           24 :                                                     initial_values[i]);
    5093              :             }
    5094        25775 :           op = initial_values[i];
    5095              :         }
    5096              : 
    5097              :       /* Create 'vect_ = {op0,op1,...,opn}'.  */
    5098       182784 :       number_of_places_left_in_vector--;
    5099       182784 :       elts[nunits - number_of_places_left_in_vector - 1] = op;
    5100       182784 :       if (!CONSTANT_CLASS_P (op))
    5101         2499 :         constant_p = false;
    5102              : 
    5103       182784 :       if (number_of_places_left_in_vector == 0)
    5104              :         {
    5105        23405 :           tree init;
    5106        46810 :           if (constant_p && !neutral_op
    5107        46516 :               ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type), nunits)
    5108        23405 :               : known_eq (TYPE_VECTOR_SUBPARTS (vector_type), nunits))
    5109              :             /* Build the vector directly from ELTS.  */
    5110        23405 :             init = gimple_build_vector (&ctor_seq, &elts);
    5111            0 :           else if (neutral_op)
    5112              :             {
    5113              :               /* Build a vector of the neutral value and shift the
    5114              :                  other elements into place.  */
    5115            0 :               init = gimple_build_vector_from_val (&ctor_seq, vector_type,
    5116              :                                                    neutral_op);
    5117            0 :               int k = nunits;
    5118            0 :               while (k > 0 && operand_equal_p (elts[k - 1], neutral_op))
    5119            0 :                 k -= 1;
    5120            0 :               while (k > 0)
    5121              :                 {
    5122            0 :                   k -= 1;
    5123            0 :                   init = gimple_build (&ctor_seq, CFN_VEC_SHL_INSERT,
    5124            0 :                                        vector_type, init, elts[k]);
    5125              :                 }
    5126              :             }
    5127              :           else
    5128              :             {
    5129              :               /* First time round, duplicate ELTS to fill the
    5130              :                  required number of vectors.  */
    5131            0 :               duplicate_and_interleave (loop_vinfo, &ctor_seq, vector_type,
    5132              :                                         elts, number_of_vectors, *vec_oprnds);
    5133            0 :               break;
    5134              :             }
    5135        23405 :           vec_oprnds->quick_push (init);
    5136              : 
    5137        23405 :           number_of_places_left_in_vector = nunits;
    5138        23405 :           elts.new_vector (vector_type, nunits, 1);
    5139        23405 :           elts.quick_grow (nunits);
    5140        23405 :           constant_p = true;
    5141              :         }
    5142              :     }
    5143        21821 :   if (ctor_seq != NULL)
    5144          461 :     vect_emit_reduction_init_stmts (loop_vinfo, reduc_info, ctor_seq);
    5145        21821 : }
    5146              : 
    5147              : vect_reduc_info
    5148       213477 : info_for_reduction (loop_vec_info loop_vinfo, slp_tree node)
    5149              : {
    5150       213477 :   gcc_assert (node);
    5151       213477 :   if (node->cycle_info.id == -1)
    5152              :     return NULL;
    5153       208677 :   return loop_vinfo->reduc_infos[node->cycle_info.id];
    5154              : }
    5155              : 
    5156              : /* See if LOOP_VINFO is an epilogue loop whose main loop had a reduction that
    5157              :    REDUC_INFO can build on.  Adjust REDUC_INFO and return true if so, otherwise
    5158              :    return false.  */
    5159              : 
    5160              : static bool
    5161        21379 : vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
    5162              :                                 vect_reduc_info reduc_info, tree vectype)
    5163              : {
    5164        21379 :   loop_vec_info main_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
    5165        21379 :   if (!main_loop_vinfo)
    5166              :     return false;
    5167              : 
    5168         4497 :   if (VECT_REDUC_INFO_TYPE (reduc_info) != TREE_CODE_REDUCTION)
    5169              :     return false;
    5170              : 
    5171              :   /* We are not set up to handle vector bools when they are not mapped
    5172              :      to vector integer data types.  */
    5173         4479 :   if (VECTOR_BOOLEAN_TYPE_P (vectype)
    5174         4531 :       && GET_MODE_CLASS (TYPE_MODE (vectype)) != MODE_VECTOR_INT)
    5175              :     return false;
    5176              : 
    5177         4477 :   unsigned int num_phis = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).length ();
    5178         4477 :   auto_vec<tree, 16> main_loop_results (num_phis);
    5179         4477 :   auto_vec<tree, 16> initial_values (num_phis);
    5180         4477 :   if (edge main_loop_edge = loop_vinfo->main_loop_edge)
    5181              :     {
    5182              :       /* The epilogue loop can be entered either from the main loop or
    5183              :          from an earlier guard block.  */
    5184         4328 :       edge skip_edge = loop_vinfo->skip_main_loop_edge;
    5185        17332 :       for (tree incoming_value : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info))
    5186              :         {
    5187              :           /* Look for:
    5188              : 
    5189              :                INCOMING_VALUE = phi<MAIN_LOOP_RESULT(main loop),
    5190              :                                     INITIAL_VALUE(guard block)>.  */
    5191         4348 :           gcc_assert (TREE_CODE (incoming_value) == SSA_NAME);
    5192              : 
    5193         4348 :           gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (incoming_value));
    5194         4348 :           gcc_assert (gimple_bb (phi) == main_loop_edge->dest);
    5195              : 
    5196         4348 :           tree from_main_loop = PHI_ARG_DEF_FROM_EDGE (phi, main_loop_edge);
    5197         4348 :           tree from_skip = PHI_ARG_DEF_FROM_EDGE (phi, skip_edge);
    5198              : 
    5199         4348 :           main_loop_results.quick_push (from_main_loop);
    5200         4348 :           initial_values.quick_push (from_skip);
    5201              :         }
    5202              :     }
    5203              :   else
    5204              :     /* The main loop dominates the epilogue loop.  */
    5205          149 :     main_loop_results.splice (VECT_REDUC_INFO_INITIAL_VALUES (reduc_info));
    5206              : 
    5207              :   /* See if the main loop has the kind of accumulator we need.  */
    5208         4477 :   vect_reusable_accumulator *accumulator
    5209         4477 :     = main_loop_vinfo->reusable_accumulators.get (main_loop_results[0]);
    5210         4477 :   if (!accumulator
    5211         8942 :       || num_phis != VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).length ()
    5212        13419 :       || !std::equal (main_loop_results.begin (), main_loop_results.end (),
    5213              :                       VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).begin ()))
    5214              :     return false;
    5215              : 
    5216              :   /* Handle the case where we can reduce wider vectors to narrower ones.  */
    5217         4471 :   tree old_vectype = TREE_TYPE (accumulator->reduc_input);
    5218         4471 :   unsigned HOST_WIDE_INT m;
    5219         4471 :   if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (old_vectype),
    5220         4471 :                             TYPE_VECTOR_SUBPARTS (vectype), &m))
    5221            0 :     return false;
    5222              :   /* Check the intermediate vector types and operations are available.  */
    5223         4471 :   tree prev_vectype = old_vectype;
    5224         4471 :   poly_uint64 intermediate_nunits = TYPE_VECTOR_SUBPARTS (old_vectype);
    5225        13039 :   while (known_gt (intermediate_nunits, TYPE_VECTOR_SUBPARTS (vectype)))
    5226              :     {
    5227         4621 :       intermediate_nunits = exact_div (intermediate_nunits, 2);
    5228         4621 :       tree intermediate_vectype = get_related_vectype_for_scalar_type
    5229         4621 :         (TYPE_MODE (vectype), TREE_TYPE (vectype), intermediate_nunits);
    5230         9242 :       if (!intermediate_vectype
    5231         4621 :           || !directly_supported_p (VECT_REDUC_INFO_CODE (reduc_info),
    5232              :                                     intermediate_vectype)
    5233         8718 :           || !can_vec_extract (TYPE_MODE (prev_vectype),
    5234         8202 :                                TYPE_MODE (intermediate_vectype)).exists ()
    5235              :           /* We could actually extract VLA vectors but bit_field_ref and
    5236              :              friends aren't yet prepared for it.  */
    5237         8718 :           || !TYPE_VECTOR_SUBPARTS (intermediate_vectype).is_constant ())
    5238              :         return false;
    5239              :       prev_vectype = intermediate_vectype;
    5240              :     }
    5241              : 
    5242              :   /* Non-SLP reductions might apply an adjustment after the reduction
    5243              :      operation, in order to simplify the initialization of the accumulator.
    5244              :      If the epilogue loop carries on from where the main loop left off,
    5245              :      it should apply the same adjustment to the final reduction result.
    5246              : 
    5247              :      If the epilogue loop can also be entered directly (rather than via
    5248              :      the main loop), we need to be able to handle that case in the same way,
    5249              :      with the same adjustment.  (In principle we could add a PHI node
    5250              :      to select the correct adjustment, but in practice that shouldn't be
    5251              :      necessary.)  */
    5252         3947 :   tree main_adjustment
    5253         3947 :     = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (accumulator->reduc_info);
    5254         3947 :   if (loop_vinfo->main_loop_edge && main_adjustment)
    5255              :     {
    5256         3336 :       gcc_assert (num_phis == 1);
    5257         3336 :       tree initial_value = initial_values[0];
    5258              :       /* Check that we can use INITIAL_VALUE as the adjustment and
    5259              :          initialize the accumulator with a neutral value instead.  */
    5260         3336 :       if (!operand_equal_p (initial_value, main_adjustment))
    5261              :         return false;
    5262         3326 :       initial_values[0] = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
    5263              :     }
    5264         3937 :   VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info) = main_adjustment;
    5265         3937 :   VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).truncate (0);
    5266         3937 :   VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).splice (initial_values);
    5267         3937 :   VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info) = accumulator;
    5268         3937 :   return true;
    5269         4477 : }
    5270              : 
    5271              : /* Reduce the vector VEC_DEF down to VECTYPE with reduction operation
    5272              :    CODE emitting stmts before GSI.  Returns a vector def of VECTYPE.  */
    5273              : 
    5274              : static tree
    5275         3981 : vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
    5276              :                             gimple_seq *seq)
    5277              : {
    5278         3981 :   gcc_assert (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (vec_def))
    5279              :               || (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (vec_def)))
    5280              :                   == MODE_VECTOR_INT));
    5281         3981 :   unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def)).to_constant ();
    5282         3981 :   unsigned nunits1 = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
    5283         3981 :   tree stype = TREE_TYPE (vectype);
    5284         3981 :   tree new_temp = vec_def;
    5285         8105 :   while (nunits > nunits1)
    5286              :     {
    5287         4124 :       nunits /= 2;
    5288         4124 :       tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
    5289         4124 :                                                            stype, nunits);
    5290         4124 :       unsigned int bitsize = tree_to_uhwi (TYPE_SIZE (vectype1));
    5291              : 
    5292              :       /* The target has to make sure we support lowpart/highpart
    5293              :          extraction, either via direct vector extract or through
    5294              :          an integer mode punning.  */
    5295         4124 :       tree dst1, dst2;
    5296         4124 :       gimple *epilog_stmt;
    5297         4124 :       if (can_vec_extract (TYPE_MODE (TREE_TYPE (new_temp)),
    5298         8248 :                            TYPE_MODE (vectype1)).exists ())
    5299              :         {
    5300              :           /* Extract sub-vectors directly once vec_extract becomes
    5301              :              a conversion optab.  */
    5302         4124 :           dst1 = make_ssa_name (vectype1);
    5303         4124 :           epilog_stmt
    5304         8248 :               = gimple_build_assign (dst1, BIT_FIELD_REF,
    5305              :                                      build3 (BIT_FIELD_REF, vectype1,
    5306         4124 :                                              new_temp, TYPE_SIZE (vectype1),
    5307              :                                              bitsize_int (0)));
    5308         4124 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5309         4124 :           dst2 =  make_ssa_name (vectype1);
    5310         4124 :           epilog_stmt
    5311         4124 :               = gimple_build_assign (dst2, BIT_FIELD_REF,
    5312              :                                      build3 (BIT_FIELD_REF, vectype1,
    5313         4124 :                                              new_temp, TYPE_SIZE (vectype1),
    5314         4124 :                                              bitsize_int (bitsize)));
    5315         4124 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5316              :         }
    5317              :       else
    5318              :         {
    5319              :           /* Extract via punning to appropriately sized integer mode
    5320              :              vector.  */
    5321            0 :           tree eltype = build_nonstandard_integer_type (bitsize, 1);
    5322            0 :           tree etype = build_vector_type (eltype, 2);
    5323            0 :           gcc_assert (can_vec_extract (TYPE_MODE (etype),
    5324              :                                        TYPE_MODE (eltype)).exists ());
    5325            0 :           tree tem = make_ssa_name (etype);
    5326            0 :           epilog_stmt = gimple_build_assign (tem, VIEW_CONVERT_EXPR,
    5327              :                                              build1 (VIEW_CONVERT_EXPR,
    5328              :                                                      etype, new_temp));
    5329            0 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5330            0 :           new_temp = tem;
    5331            0 :           tem = make_ssa_name (eltype);
    5332            0 :           epilog_stmt
    5333            0 :               = gimple_build_assign (tem, BIT_FIELD_REF,
    5334              :                                      build3 (BIT_FIELD_REF, eltype,
    5335            0 :                                              new_temp, TYPE_SIZE (eltype),
    5336              :                                              bitsize_int (0)));
    5337            0 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5338            0 :           dst1 = make_ssa_name (vectype1);
    5339            0 :           epilog_stmt = gimple_build_assign (dst1, VIEW_CONVERT_EXPR,
    5340              :                                              build1 (VIEW_CONVERT_EXPR,
    5341              :                                                      vectype1, tem));
    5342            0 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5343            0 :           tem = make_ssa_name (eltype);
    5344            0 :           epilog_stmt
    5345            0 :               = gimple_build_assign (tem, BIT_FIELD_REF,
    5346              :                                      build3 (BIT_FIELD_REF, eltype,
    5347            0 :                                              new_temp, TYPE_SIZE (eltype),
    5348            0 :                                              bitsize_int (bitsize)));
    5349            0 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5350            0 :           dst2 =  make_ssa_name (vectype1);
    5351            0 :           epilog_stmt = gimple_build_assign (dst2, VIEW_CONVERT_EXPR,
    5352              :                                              build1 (VIEW_CONVERT_EXPR,
    5353              :                                                      vectype1, tem));
    5354            0 :           gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5355              :         }
    5356              : 
    5357         4124 :       new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
    5358              :     }
    5359         3981 :   if (!useless_type_conversion_p (vectype, TREE_TYPE (new_temp)))
    5360              :     {
    5361           46 :       tree dst3 = make_ssa_name (vectype);
    5362           46 :       gimple *epilog_stmt = gimple_build_assign (dst3, VIEW_CONVERT_EXPR,
    5363              :                                                  build1 (VIEW_CONVERT_EXPR,
    5364              :                                                          vectype, new_temp));
    5365           46 :       gimple_seq_add_stmt_without_update (seq, epilog_stmt);
    5366           46 :       new_temp = dst3;
    5367              :     }
    5368              : 
    5369         3981 :   return new_temp;
    5370              : }
    5371              : 
    5372              : /* Function vect_create_epilog_for_reduction
    5373              : 
    5374              :    Create code at the loop-epilog to finalize the result of a reduction
    5375              :    computation.
    5376              : 
    5377              :    STMT_INFO is the scalar reduction stmt that is being vectorized.
    5378              :    SLP_NODE is an SLP node containing a group of reduction statements. The
    5379              :      first one in this group is STMT_INFO.
    5380              :    SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
    5381              :    REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
    5382              :      (counting from 0)
    5383              :    LOOP_EXIT is the edge to update in the merge block.  In the case of a single
    5384              :      exit this edge is always the main loop exit.
    5385              : 
    5386              :    This function:
    5387              :    1. Completes the reduction def-use cycles.
    5388              :    2. "Reduces" each vector of partial results VECT_DEFS into a single result,
    5389              :       by calling the function specified by REDUC_FN if available, or by
    5390              :       other means (whole-vector shifts or a scalar loop).
    5391              :       The function also creates a new phi node at the loop exit to preserve
    5392              :       loop-closed form, as illustrated below.
    5393              : 
    5394              :      The flow at the entry to this function:
    5395              : 
    5396              :         loop:
    5397              :           vec_def = phi <vec_init, null>        # REDUCTION_PHI
    5398              :           VECT_DEF = vector_stmt                # vectorized form of STMT_INFO
    5399              :           s_loop = scalar_stmt                  # (scalar) STMT_INFO
    5400              :         loop_exit:
    5401              :           s_out0 = phi <s_loop>                 # (scalar) EXIT_PHI
    5402              :           use <s_out0>
    5403              :           use <s_out0>
    5404              : 
    5405              :      The above is transformed by this function into:
    5406              : 
    5407              :         loop:
    5408              :           vec_def = phi <vec_init, VECT_DEF>    # REDUCTION_PHI
    5409              :           VECT_DEF = vector_stmt                # vectorized form of STMT_INFO
    5410              :           s_loop = scalar_stmt                  # (scalar) STMT_INFO
    5411              :         loop_exit:
    5412              :           s_out0 = phi <s_loop>                 # (scalar) EXIT_PHI
    5413              :           v_out1 = phi <VECT_DEF>               # NEW_EXIT_PHI
    5414              :           v_out2 = reduce <v_out1>
    5415              :           s_out3 = extract_field <v_out2, 0>
    5416              :           s_out4 = adjust_result <s_out3>
    5417              :           use <s_out4>
    5418              :           use <s_out4>
    5419              : */
    5420              : 
    5421              : static void
    5422        22094 : vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
    5423              :                                   stmt_vec_info stmt_info,
    5424              :                                   slp_tree slp_node,
    5425              :                                   slp_instance slp_node_instance,
    5426              :                                   edge loop_exit)
    5427              : {
    5428        22094 :   vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
    5429        22094 :   code_helper code = VECT_REDUC_INFO_CODE (reduc_info);
    5430        22094 :   internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
    5431        22094 :   tree vectype;
    5432        22094 :   machine_mode mode;
    5433        22094 :   basic_block exit_bb;
    5434        22094 :   gimple *new_phi = NULL, *phi = NULL;
    5435        22094 :   gimple_stmt_iterator exit_gsi;
    5436        22094 :   tree new_temp = NULL_TREE, new_name, new_scalar_dest;
    5437        22094 :   gimple *epilog_stmt = NULL;
    5438        22094 :   gimple *exit_phi;
    5439        22094 :   tree def;
    5440        22094 :   tree orig_name, scalar_result;
    5441        22094 :   imm_use_iterator imm_iter;
    5442        22094 :   use_operand_p use_p;
    5443        22094 :   gimple *use_stmt;
    5444        22094 :   auto_vec<tree> reduc_inputs;
    5445        22094 :   int j, i;
    5446        22094 :   vec<tree> &scalar_results = VECT_REDUC_INFO_SCALAR_RESULTS (reduc_info);
    5447        22094 :   unsigned int k;
    5448              :   /* SLP reduction without reduction chain, e.g.,
    5449              :      # a1 = phi <a2, a0>
    5450              :      # b1 = phi <b2, b0>
    5451              :      a2 = operation (a1)
    5452              :      b2 = operation (b1)  */
    5453        22094 :   const bool slp_reduc = !reduc_info->is_reduc_chain;
    5454        22094 :   tree induction_index = NULL_TREE;
    5455              : 
    5456        22094 :   unsigned int group_size = SLP_TREE_LANES (slp_node);
    5457              : 
    5458        22094 :   bool double_reduc = false;
    5459        22094 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    5460        22094 :   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
    5461              :     {
    5462            0 :       double_reduc = true;
    5463            0 :       gcc_assert (slp_reduc);
    5464              :     }
    5465              : 
    5466        22094 :   vectype = VECT_REDUC_INFO_VECTYPE (reduc_info);
    5467        22094 :   gcc_assert (vectype);
    5468        22094 :   mode = TYPE_MODE (vectype);
    5469              : 
    5470        22094 :   tree induc_val = NULL_TREE;
    5471        22094 :   tree adjustment_def = NULL;
    5472              :   /* Optimize: for induction condition reduction, if we can't use zero
    5473              :      for induc_val, use initial_def.  */
    5474        22094 :   if (VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
    5475           62 :     induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
    5476        22032 :   else if (double_reduc)
    5477              :     ;
    5478              :   else
    5479        22032 :     adjustment_def = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info);
    5480              : 
    5481        22094 :   stmt_vec_info single_live_out_stmt[] = { stmt_info };
    5482        22094 :   array_slice<const stmt_vec_info> live_out_stmts = single_live_out_stmt;
    5483        22094 :   if (slp_reduc)
    5484              :     /* All statements produce live-out values.  */
    5485        43726 :     live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
    5486              : 
    5487        22094 :   unsigned vec_num
    5488        22094 :     = SLP_TREE_VEC_DEFS (slp_node_instance->reduc_phis).length ();
    5489              : 
    5490              :   /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
    5491              :      which is updated with the current index of the loop for every match of
    5492              :      the original loop's cond_expr (VEC_STMT).  This results in a vector
    5493              :      containing the last time the condition passed for that vector lane.
    5494              :      The first match will be a 1 to allow 0 to be used for non-matching
    5495              :      indexes.  If there are no matches at all then the vector will be all
    5496              :      zeroes.
    5497              : 
    5498              :      PR92772: This algorithm is broken for architectures that support
    5499              :      masked vectors, but do not provide fold_extract_last.  */
    5500        22094 :   if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION)
    5501              :     {
    5502           99 :       gcc_assert (!double_reduc);
    5503           99 :       auto_vec<std::pair<tree, bool>, 2> ccompares;
    5504           99 :       slp_tree cond_node = slp_node_instance->root;
    5505          207 :       while (cond_node != slp_node_instance->reduc_phis)
    5506              :         {
    5507          108 :           stmt_vec_info cond_info = SLP_TREE_REPRESENTATIVE (cond_node);
    5508          108 :           if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
    5509              :             {
    5510          108 :               gimple *vec_stmt
    5511          108 :                 = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (cond_node)[0]);
    5512          108 :               gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
    5513          108 :               ccompares.safe_push
    5514          108 :                 (std::make_pair (gimple_assign_rhs1 (vec_stmt),
    5515          108 :                                  SLP_TREE_REDUC_IDX (cond_node) == 2));
    5516              :             }
    5517          108 :           int slp_reduc_idx = SLP_TREE_REDUC_IDX (cond_node);
    5518          108 :           cond_node = SLP_TREE_CHILDREN (cond_node)[slp_reduc_idx];
    5519              :         }
    5520           99 :       gcc_assert (ccompares.length () != 0);
    5521              : 
    5522           99 :       tree indx_before_incr, indx_after_incr;
    5523           99 :       poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
    5524           99 :       int scalar_precision
    5525           99 :         = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype)));
    5526           99 :       tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
    5527           99 :       tree cr_index_vector_type = get_related_vectype_for_scalar_type
    5528           99 :         (TYPE_MODE (vectype), cr_index_scalar_type,
    5529              :          TYPE_VECTOR_SUBPARTS (vectype));
    5530              : 
    5531              :       /* First we create a simple vector induction variable which starts
    5532              :          with the values {1,2,3,...} (SERIES_VECT) and increments by the
    5533              :          vector size (STEP).  */
    5534              : 
    5535              :       /* Create a {1,2,3,...} vector.  */
    5536           99 :       tree series_vect = build_index_vector (cr_index_vector_type, 1, 1);
    5537              : 
    5538              :       /* Create a vector of the step value.  */
    5539           99 :       tree step = build_int_cst (cr_index_scalar_type, nunits_out);
    5540           99 :       tree vec_step = build_vector_from_val (cr_index_vector_type, step);
    5541              : 
    5542              :       /* Create an induction variable.  */
    5543           99 :       gimple_stmt_iterator incr_gsi;
    5544           99 :       bool insert_after;
    5545           99 :       vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
    5546              :                                   &incr_gsi, &insert_after);
    5547           99 :       create_iv (series_vect, PLUS_EXPR, vec_step, NULL_TREE, loop, &incr_gsi,
    5548              :                  insert_after, &indx_before_incr, &indx_after_incr);
    5549              : 
    5550              :       /* Next create a new phi node vector (NEW_PHI_TREE) which starts
    5551              :          filled with zeros (VEC_ZERO).  */
    5552              : 
    5553              :       /* Create a vector of 0s.  */
    5554           99 :       tree zero = build_zero_cst (cr_index_scalar_type);
    5555           99 :       tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
    5556              : 
    5557              :       /* Create a vector phi node.  */
    5558           99 :       tree new_phi_tree = make_ssa_name (cr_index_vector_type);
    5559           99 :       new_phi = create_phi_node (new_phi_tree, loop->header);
    5560           99 :       add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
    5561              :                    loop_preheader_edge (loop), UNKNOWN_LOCATION);
    5562              : 
    5563              :       /* Now take the condition from the loops original cond_exprs
    5564              :          and produce a new cond_exprs (INDEX_COND_EXPR) which for
    5565              :          every match uses values from the induction variable
    5566              :          (INDEX_BEFORE_INCR) otherwise uses values from the phi node
    5567              :          (NEW_PHI_TREE).
    5568              :          Finally, we update the phi (NEW_PHI_TREE) to take the value of
    5569              :          the new cond_expr (INDEX_COND_EXPR).  */
    5570           99 :       gimple_seq stmts = NULL;
    5571          306 :       for (int i = ccompares.length () - 1; i != -1; --i)
    5572              :         {
    5573          108 :           tree ccompare = ccompares[i].first;
    5574          108 :           if (ccompares[i].second)
    5575           81 :             new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
    5576              :                                          cr_index_vector_type,
    5577              :                                          ccompare,
    5578              :                                          indx_before_incr, new_phi_tree);
    5579              :           else
    5580           27 :             new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
    5581              :                                          cr_index_vector_type,
    5582              :                                          ccompare,
    5583              :                                          new_phi_tree, indx_before_incr);
    5584              :         }
    5585           99 :       gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
    5586              : 
    5587              :       /* Update the phi with the vec cond.  */
    5588           99 :       induction_index = new_phi_tree;
    5589           99 :       add_phi_arg (as_a <gphi *> (new_phi), induction_index,
    5590              :                    loop_latch_edge (loop), UNKNOWN_LOCATION);
    5591           99 :     }
    5592              : 
    5593              :   /* 2. Create epilog code.
    5594              :         The reduction epilog code operates across the elements of the vector
    5595              :         of partial results computed by the vectorized loop.
    5596              :         The reduction epilog code consists of:
    5597              : 
    5598              :         step 1: compute the scalar result in a vector (v_out2)
    5599              :         step 2: extract the scalar result (s_out3) from the vector (v_out2)
    5600              :         step 3: adjust the scalar result (s_out3) if needed.
    5601              : 
    5602              :         Step 1 can be accomplished using one the following three schemes:
    5603              :           (scheme 1) using reduc_fn, if available.
    5604              :           (scheme 2) using whole-vector shifts, if available.
    5605              :           (scheme 3) using a scalar loop. In this case steps 1+2 above are
    5606              :                      combined.
    5607              : 
    5608              :           The overall epilog code looks like this:
    5609              : 
    5610              :           s_out0 = phi <s_loop>         # original EXIT_PHI
    5611              :           v_out1 = phi <VECT_DEF>       # NEW_EXIT_PHI
    5612              :           v_out2 = reduce <v_out1>              # step 1
    5613              :           s_out3 = extract_field <v_out2, 0>    # step 2
    5614              :           s_out4 = adjust_result <s_out3>       # step 3
    5615              : 
    5616              :           (step 3 is optional, and steps 1 and 2 may be combined).
    5617              :           Lastly, the uses of s_out0 are replaced by s_out4.  */
    5618              : 
    5619              : 
    5620              :   /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
    5621              :          v_out1 = phi <VECT_DEF>
    5622              :          Store them in NEW_PHIS.  */
    5623              :   /* We need to reduce values in all exits.  */
    5624        22094 :   exit_bb = loop_exit->dest;
    5625        22094 :   exit_gsi = gsi_after_labels (exit_bb);
    5626        22094 :   reduc_inputs.create (vec_num);
    5627        67876 :   for (unsigned i = 0; i < vec_num; i++)
    5628              :     {
    5629        23688 :       gimple_seq stmts = NULL;
    5630        23688 :       def = vect_get_slp_vect_def (slp_node, i);
    5631        23688 :       tree new_def = copy_ssa_name (def);
    5632        23688 :       phi = create_phi_node (new_def, exit_bb);
    5633        23688 :       if (LOOP_VINFO_MAIN_EXIT (loop_vinfo) == loop_exit)
    5634        23661 :         SET_PHI_ARG_DEF (phi, loop_exit->dest_idx, def);
    5635              :       else
    5636              :         {
    5637           57 :           for (unsigned k = 0; k < gimple_phi_num_args (phi); k++)
    5638           30 :             SET_PHI_ARG_DEF (phi, k, def);
    5639              :         }
    5640        23688 :       new_def = gimple_convert (&stmts, vectype, new_def);
    5641        23688 :       reduc_inputs.quick_push (new_def);
    5642        23688 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5643              :     }
    5644              : 
    5645              :   /* 2.2 Get the original scalar reduction variable as defined in the loop.
    5646              :          In case STMT is a "pattern-stmt" (i.e. - it represents a reduction
    5647              :          pattern), the scalar-def is taken from the original stmt that the
    5648              :          pattern-stmt (STMT) replaces.  */
    5649              : 
    5650        22938 :   tree scalar_dest = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
    5651        22094 :   tree scalar_type = TREE_TYPE (scalar_dest);
    5652        22094 :   scalar_results.truncate (0);
    5653        22094 :   scalar_results.reserve_exact (group_size);
    5654        22094 :   new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
    5655              : 
    5656              :   /* True if we should implement SLP_REDUC using native reduction operations
    5657              :      instead of scalar operations.  */
    5658        22094 :   const bool direct_slp_reduc
    5659        22094 :     = (reduc_fn != IFN_LAST
    5660        22094 :        && slp_reduc
    5661        22094 :        && !TYPE_VECTOR_SUBPARTS (vectype).is_constant ());
    5662              : 
    5663              :   /* If signed overflow is undefined we might need to perform reduction
    5664              :      computations in an unsigned type.  */
    5665        22094 :   tree compute_vectype = vectype;
    5666        22094 :   if (ANY_INTEGRAL_TYPE_P (vectype)
    5667        14961 :       && TYPE_OVERFLOW_UNDEFINED (vectype)
    5668         5512 :       && code.is_tree_code ()
    5669        27606 :       && arith_code_with_undefined_signed_overflow ((tree_code) code))
    5670         4076 :     compute_vectype = unsigned_type_for (vectype);
    5671              : 
    5672              :   /* In case of reduction chain, e.g.,
    5673              :      # a1 = phi <a3, a0>
    5674              :      a2 = operation (a1)
    5675              :      a3 = operation (a2),
    5676              : 
    5677              :      we may end up with more than one vector result.  Here we reduce them
    5678              :      to one vector.
    5679              : 
    5680              :      The same is true for a SLP reduction, e.g.,
    5681              :      # a1 = phi <a2, a0>
    5682              :      # b1 = phi <b2, b0>
    5683              :      a2 = operation (a1)
    5684              :      b2 = operation (a2),
    5685              : 
    5686              :      where we can end up with more than one vector as well.  We can
    5687              :      easily accumulate vectors when the number of vector elements is
    5688              :      a multiple of the SLP group size.
    5689              : 
    5690              :      The same is true if we couldn't use a single defuse cycle.  */
    5691        22094 :   if ((!slp_reduc
    5692              :        || direct_slp_reduc
    5693              :        || (slp_reduc
    5694        22094 :            && constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype), group_size)))
    5695        44188 :       && reduc_inputs.length () > 1)
    5696              :     {
    5697          550 :       gimple_seq stmts = NULL;
    5698          550 :       tree single_input = reduc_inputs[0];
    5699          550 :       if (compute_vectype != vectype)
    5700          167 :         single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
    5701              :                                      compute_vectype, single_input);
    5702         1989 :       for (k = 1; k < reduc_inputs.length (); k++)
    5703              :         {
    5704         1439 :           tree input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
    5705         1439 :                                      compute_vectype, reduc_inputs[k]);
    5706         1439 :           single_input = gimple_build (&stmts, code, compute_vectype,
    5707              :                                        single_input, input);
    5708              :         }
    5709          550 :       if (compute_vectype != vectype)
    5710          167 :         single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
    5711              :                                      vectype, single_input);
    5712          550 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5713              : 
    5714          550 :       reduc_inputs.truncate (0);
    5715          550 :       reduc_inputs.safe_push (single_input);
    5716              :     }
    5717              : 
    5718        22094 :   tree orig_reduc_input = reduc_inputs[0];
    5719              : 
    5720              :   /* If this loop is an epilogue loop that can be skipped after the
    5721              :      main loop, we can only share a reduction operation between the
    5722              :      main loop and the epilogue if we put it at the target of the
    5723              :      skip edge.
    5724              : 
    5725              :      We can still reuse accumulators if this check fails.  Doing so has
    5726              :      the minor(?) benefit of making the epilogue loop's scalar result
    5727              :      independent of the main loop's scalar result.  */
    5728        22094 :   bool unify_with_main_loop_p = false;
    5729        22094 :   if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
    5730         3937 :       && loop_vinfo->skip_this_loop_edge
    5731         3771 :       && single_succ_p (exit_bb)
    5732        22111 :       && single_succ (exit_bb) == loop_vinfo->skip_this_loop_edge->dest)
    5733              :     {
    5734           17 :       unify_with_main_loop_p = true;
    5735              : 
    5736           17 :       basic_block reduc_block = loop_vinfo->skip_this_loop_edge->dest;
    5737           17 :       reduc_inputs[0] = make_ssa_name (vectype);
    5738           17 :       gphi *new_phi = create_phi_node (reduc_inputs[0], reduc_block);
    5739           17 :       add_phi_arg (new_phi, orig_reduc_input, single_succ_edge (exit_bb),
    5740              :                    UNKNOWN_LOCATION);
    5741           17 :       add_phi_arg (new_phi,
    5742           17 :                    VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)->reduc_input,
    5743              :                    loop_vinfo->skip_this_loop_edge, UNKNOWN_LOCATION);
    5744           17 :       exit_gsi = gsi_after_labels (reduc_block);
    5745              :     }
    5746              : 
    5747              :   /* Shouldn't be used beyond this point.  */
    5748        22094 :   exit_bb = nullptr;
    5749              : 
    5750              :   /* If we are operating on a mask vector and do not support direct mask
    5751              :      reduction, work on a bool data vector instead of a mask vector.  */
    5752        22094 :   if (VECTOR_BOOLEAN_TYPE_P (vectype)
    5753          235 :       && VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info)
    5754        22279 :       && vectype != VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info))
    5755              :     {
    5756          185 :       compute_vectype = vectype = VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info);
    5757          185 :       gimple_seq stmts = NULL;
    5758          378 :       for (unsigned i = 0; i < reduc_inputs.length (); ++i)
    5759          386 :         reduc_inputs[i] = gimple_build (&stmts, VEC_COND_EXPR, vectype,
    5760          193 :                                         reduc_inputs[i],
    5761              :                                         build_one_cst (vectype),
    5762              :                                         build_zero_cst (vectype));
    5763          185 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5764              :     }
    5765              : 
    5766        22094 :   if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
    5767           99 :       && reduc_fn != IFN_LAST)
    5768              :     {
    5769              :       /* For condition reductions, we have a vector (REDUC_INPUTS 0) containing
    5770              :          various data values where the condition matched and another vector
    5771              :          (INDUCTION_INDEX) containing all the indexes of those matches.  We
    5772              :          need to extract the last matching index (which will be the index with
    5773              :          highest value) and use this to index into the data vector.
    5774              :          For the case where there were no matches, the data vector will contain
    5775              :          all default values and the index vector will be all zeros.  */
    5776              : 
    5777              :       /* Get various versions of the type of the vector of indexes.  */
    5778           14 :       tree index_vec_type = TREE_TYPE (induction_index);
    5779           14 :       gcc_checking_assert (TYPE_UNSIGNED (index_vec_type));
    5780           14 :       tree index_scalar_type = TREE_TYPE (index_vec_type);
    5781           14 :       tree index_vec_cmp_type = truth_type_for (index_vec_type);
    5782              : 
    5783              :       /* Get an unsigned integer version of the type of the data vector.  */
    5784           14 :       int scalar_precision
    5785           14 :         = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
    5786           14 :       tree scalar_type_unsigned = make_unsigned_type (scalar_precision);
    5787           14 :       tree vectype_unsigned = get_same_sized_vectype (scalar_type_unsigned,
    5788              :                                                 vectype);
    5789              : 
    5790              :       /* First we need to create a vector (ZERO_VEC) of zeros and another
    5791              :          vector (MAX_INDEX_VEC) filled with the last matching index, which we
    5792              :          can create using a MAX reduction and then expanding.
    5793              :          In the case where the loop never made any matches, the max index will
    5794              :          be zero.  */
    5795              : 
    5796              :       /* Vector of {0, 0, 0,...}.  */
    5797           14 :       tree zero_vec = build_zero_cst (vectype);
    5798              : 
    5799              :       /* Find maximum value from the vector of found indexes.  */
    5800           14 :       tree max_index = make_ssa_name (index_scalar_type);
    5801           14 :       gcall *max_index_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
    5802              :                                                           1, induction_index);
    5803           14 :       gimple_call_set_lhs (max_index_stmt, max_index);
    5804           14 :       gsi_insert_before (&exit_gsi, max_index_stmt, GSI_SAME_STMT);
    5805              : 
    5806              :       /* Vector of {max_index, max_index, max_index,...}.  */
    5807           14 :       tree max_index_vec = make_ssa_name (index_vec_type);
    5808           14 :       tree max_index_vec_rhs = build_vector_from_val (index_vec_type,
    5809              :                                                       max_index);
    5810           14 :       gimple *max_index_vec_stmt = gimple_build_assign (max_index_vec,
    5811              :                                                         max_index_vec_rhs);
    5812           14 :       gsi_insert_before (&exit_gsi, max_index_vec_stmt, GSI_SAME_STMT);
    5813              : 
    5814              :       /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
    5815              :          with the vector (INDUCTION_INDEX) of found indexes, choosing values
    5816              :          from the data vector (REDUC_INPUTS 0) for matches, 0 (ZERO_VEC)
    5817              :          otherwise.  Only one value should match, resulting in a vector
    5818              :          (VEC_COND) with one data value and the rest zeros.
    5819              :          In the case where the loop never made any matches, every index will
    5820              :          match, resulting in a vector with all data values (which will all be
    5821              :          the default value).  */
    5822              : 
    5823              :       /* Compare the max index vector to the vector of found indexes to find
    5824              :          the position of the max value.  */
    5825           14 :       tree vec_compare = make_ssa_name (index_vec_cmp_type);
    5826           14 :       gimple *vec_compare_stmt = gimple_build_assign (vec_compare, EQ_EXPR,
    5827              :                                                       induction_index,
    5828              :                                                       max_index_vec);
    5829           14 :       gsi_insert_before (&exit_gsi, vec_compare_stmt, GSI_SAME_STMT);
    5830              : 
    5831              :       /* Use the compare to choose either values from the data vector or
    5832              :          zero.  */
    5833           14 :       tree vec_cond = make_ssa_name (vectype);
    5834           14 :       gimple *vec_cond_stmt = gimple_build_assign (vec_cond, VEC_COND_EXPR,
    5835              :                                                    vec_compare,
    5836           14 :                                                    reduc_inputs[0],
    5837              :                                                    zero_vec);
    5838           14 :       gsi_insert_before (&exit_gsi, vec_cond_stmt, GSI_SAME_STMT);
    5839              : 
    5840              :       /* Finally we need to extract the data value from the vector (VEC_COND)
    5841              :          into a scalar (MATCHED_DATA_REDUC).  Logically we want to do a OR
    5842              :          reduction, but because this doesn't exist, we can use a MAX reduction
    5843              :          instead.  The data value might be signed or a float so we need to cast
    5844              :          it first.
    5845              :          In the case where the loop never made any matches, the data values are
    5846              :          all identical, and so will reduce down correctly.  */
    5847              : 
    5848              :       /* Make the matched data values unsigned.  */
    5849           14 :       tree vec_cond_cast = make_ssa_name (vectype_unsigned);
    5850           14 :       tree vec_cond_cast_rhs = build1 (VIEW_CONVERT_EXPR, vectype_unsigned,
    5851              :                                        vec_cond);
    5852           14 :       gimple *vec_cond_cast_stmt = gimple_build_assign (vec_cond_cast,
    5853              :                                                         VIEW_CONVERT_EXPR,
    5854              :                                                         vec_cond_cast_rhs);
    5855           14 :       gsi_insert_before (&exit_gsi, vec_cond_cast_stmt, GSI_SAME_STMT);
    5856              : 
    5857              :       /* Reduce down to a scalar value.  */
    5858           14 :       tree data_reduc = make_ssa_name (scalar_type_unsigned);
    5859           14 :       gcall *data_reduc_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
    5860              :                                                            1, vec_cond_cast);
    5861           14 :       gimple_call_set_lhs (data_reduc_stmt, data_reduc);
    5862           14 :       gsi_insert_before (&exit_gsi, data_reduc_stmt, GSI_SAME_STMT);
    5863              : 
    5864              :       /* Convert the reduced value back to the result type and set as the
    5865              :          result.  */
    5866           14 :       gimple_seq stmts = NULL;
    5867           14 :       new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR, scalar_type,
    5868              :                                data_reduc);
    5869           14 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5870           14 :       scalar_results.safe_push (new_temp);
    5871           14 :     }
    5872        22080 :   else if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
    5873           85 :            && reduc_fn == IFN_LAST)
    5874              :     {
    5875              :       /* Condition reduction without supported IFN_REDUC_MAX.  Generate
    5876              :          idx = 0;
    5877              :          idx_val = induction_index[0];
    5878              :          val = data_reduc[0];
    5879              :          for (idx = 0, val = init, i = 0; i < nelts; ++i)
    5880              :            if (induction_index[i] > idx_val)
    5881              :              val = data_reduc[i], idx_val = induction_index[i];
    5882              :          return val;  */
    5883              : 
    5884           85 :       tree data_eltype = TREE_TYPE (vectype);
    5885           85 :       tree idx_eltype = TREE_TYPE (TREE_TYPE (induction_index));
    5886           85 :       unsigned HOST_WIDE_INT el_size = tree_to_uhwi (TYPE_SIZE (idx_eltype));
    5887           85 :       poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index));
    5888              :       /* Enforced by vectorizable_reduction, which ensures we have target
    5889              :          support before allowing a conditional reduction on variable-length
    5890              :          vectors.  */
    5891           85 :       unsigned HOST_WIDE_INT v_size = el_size * nunits.to_constant ();
    5892           85 :       tree idx_val = NULL_TREE, val = NULL_TREE;
    5893          511 :       for (unsigned HOST_WIDE_INT off = 0; off < v_size; off += el_size)
    5894              :         {
    5895          426 :           tree old_idx_val = idx_val;
    5896          426 :           tree old_val = val;
    5897          426 :           idx_val = make_ssa_name (idx_eltype);
    5898          426 :           epilog_stmt = gimple_build_assign (idx_val, BIT_FIELD_REF,
    5899              :                                              build3 (BIT_FIELD_REF, idx_eltype,
    5900              :                                                      induction_index,
    5901          426 :                                                      bitsize_int (el_size),
    5902          426 :                                                      bitsize_int (off)));
    5903          426 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5904          426 :           val = make_ssa_name (data_eltype);
    5905          852 :           epilog_stmt = gimple_build_assign (val, BIT_FIELD_REF,
    5906              :                                              build3 (BIT_FIELD_REF,
    5907              :                                                      data_eltype,
    5908          426 :                                                      reduc_inputs[0],
    5909          426 :                                                      bitsize_int (el_size),
    5910          426 :                                                      bitsize_int (off)));
    5911          426 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5912          426 :           if (off != 0)
    5913              :             {
    5914          341 :               tree new_idx_val = idx_val;
    5915          341 :               if (off != v_size - el_size)
    5916              :                 {
    5917          256 :                   new_idx_val = make_ssa_name (idx_eltype);
    5918          256 :                   epilog_stmt = gimple_build_assign (new_idx_val,
    5919              :                                                      MAX_EXPR, idx_val,
    5920              :                                                      old_idx_val);
    5921          256 :                   gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5922              :                 }
    5923          341 :               tree cond = make_ssa_name (boolean_type_node);
    5924          341 :               epilog_stmt = gimple_build_assign (cond, GT_EXPR,
    5925              :                                                  idx_val, old_idx_val);
    5926          341 :               gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5927          341 :               tree new_val = make_ssa_name (data_eltype);
    5928          341 :               epilog_stmt = gimple_build_assign (new_val, COND_EXPR,
    5929              :                                                  cond, val, old_val);
    5930          341 :               gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5931          341 :               idx_val = new_idx_val;
    5932          341 :               val = new_val;
    5933              :             }
    5934              :         }
    5935              :       /* Convert the reduced value back to the result type and set as the
    5936              :          result.  */
    5937           85 :       gimple_seq stmts = NULL;
    5938           85 :       val = gimple_convert (&stmts, scalar_type, val);
    5939           85 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5940           85 :       scalar_results.safe_push (val);
    5941           85 :     }
    5942              : 
    5943              :   /* 2.3 Create the reduction code, using one of the three schemes described
    5944              :          above. In SLP we simply need to extract all the elements from the
    5945              :          vector (without reducing them), so we use scalar shifts.  */
    5946        21995 :   else if (reduc_fn != IFN_LAST && (!slp_reduc || group_size == 1))
    5947              :     {
    5948        20178 :       tree tmp;
    5949        20178 :       tree vec_elem_type;
    5950              : 
    5951              :       /* Case 1:  Create:
    5952              :          v_out2 = reduc_expr <v_out1>  */
    5953              : 
    5954        20178 :       if (dump_enabled_p ())
    5955         1552 :         dump_printf_loc (MSG_NOTE, vect_location,
    5956              :                          "Reduce using direct vector reduction.\n");
    5957              : 
    5958        20178 :       gimple_seq stmts = NULL;
    5959        20178 :       vec_elem_type = TREE_TYPE (vectype);
    5960        20178 :       new_temp = gimple_build (&stmts, as_combined_fn (reduc_fn),
    5961        20178 :                                vec_elem_type, reduc_inputs[0]);
    5962        20178 :       new_temp = gimple_convert (&stmts, scalar_type, new_temp);
    5963        20178 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    5964              : 
    5965        20178 :       if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
    5966           62 :           && induc_val)
    5967              :         {
    5968              :           /* Earlier we set the initial value to be a vector if induc_val
    5969              :              values.  Check the result and if it is induc_val then replace
    5970              :              with the original initial value, unless induc_val is
    5971              :              the same as initial_def already.  */
    5972           60 :           tree zcompare = make_ssa_name (boolean_type_node);
    5973           60 :           epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
    5974              :                                              new_temp, induc_val);
    5975           60 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5976           60 :           tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
    5977           60 :           tmp = make_ssa_name (new_scalar_dest);
    5978           60 :           epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
    5979              :                                              initial_def, new_temp);
    5980           60 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    5981           60 :           new_temp = tmp;
    5982              :         }
    5983              : 
    5984        20178 :       scalar_results.safe_push (new_temp);
    5985        20178 :     }
    5986         1619 :   else if (direct_slp_reduc)
    5987              :     {
    5988              :       /* Here we create one vector for each of the GROUP_SIZE results,
    5989              :          with the elements for other SLP statements replaced with the
    5990              :          neutral value.  We can then do a normal reduction on each vector.  */
    5991              : 
    5992              :       /* Enforced by vectorizable_reduction.  */
    5993              :       gcc_assert (reduc_inputs.length () == 1);
    5994              :       gcc_assert (pow2p_hwi (group_size));
    5995              : 
    5996              :       gimple_seq seq = NULL;
    5997              : 
    5998              :       /* Build a vector {0, 1, 2, ...}, with the same number of elements
    5999              :          and the same element size as VECTYPE.  */
    6000              :       tree index = build_index_vector (vectype, 0, 1);
    6001              :       tree index_type = TREE_TYPE (index);
    6002              :       tree index_elt_type = TREE_TYPE (index_type);
    6003              :       tree mask_type = truth_type_for (index_type);
    6004              : 
    6005              :       /* Create a vector that, for each element, identifies which of
    6006              :          the results should use it.  */
    6007              :       tree index_mask = build_int_cst (index_elt_type, group_size - 1);
    6008              :       index = gimple_build (&seq, BIT_AND_EXPR, index_type, index,
    6009              :                             build_vector_from_val (index_type, index_mask));
    6010              : 
    6011              :       /* Get a neutral vector value.  This is simply a splat of the neutral
    6012              :          scalar value if we have one, otherwise the initial scalar value
    6013              :          is itself a neutral value.  */
    6014              :       tree vector_identity = NULL_TREE;
    6015              :       tree neutral_op = neutral_op_for_reduction (TREE_TYPE (vectype), code,
    6016              :                                                   NULL_TREE, false);
    6017              :       if (neutral_op)
    6018              :         vector_identity = gimple_build_vector_from_val (&seq, vectype,
    6019              :                                                         neutral_op);
    6020              :       for (unsigned int i = 0; i < group_size; ++i)
    6021              :         {
    6022              :           /* If there's no universal neutral value, we can use the
    6023              :              initial scalar value from the original PHI.  This is used
    6024              :              for MIN and MAX reduction, for example.  */
    6025              :           if (!neutral_op)
    6026              :             {
    6027              :               tree scalar_value
    6028              :                 = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[i];
    6029              :               scalar_value = gimple_convert (&seq, TREE_TYPE (vectype),
    6030              :                                              scalar_value);
    6031              :               vector_identity = gimple_build_vector_from_val (&seq, vectype,
    6032              :                                                               scalar_value);
    6033              :             }
    6034              : 
    6035              :           /* Calculate the equivalent of:
    6036              : 
    6037              :              sel[j] = (index[j] == i);
    6038              : 
    6039              :              which selects the elements of REDUC_INPUTS[0] that should
    6040              :              be included in the result.  */
    6041              :           tree compare_val = build_int_cst (index_elt_type, i);
    6042              :           compare_val = build_vector_from_val (index_type, compare_val);
    6043              :           tree sel = gimple_build (&seq, EQ_EXPR, mask_type,
    6044              :                                    index, compare_val);
    6045              : 
    6046              :           /* Calculate the equivalent of:
    6047              : 
    6048              :              vec = seq ? reduc_inputs[0] : vector_identity;
    6049              : 
    6050              :              VEC is now suitable for a full vector reduction.  */
    6051              :           tree vec = gimple_build (&seq, VEC_COND_EXPR, vectype,
    6052              :                                    sel, reduc_inputs[0], vector_identity);
    6053              : 
    6054              :           /* Do the reduction and convert it to the appropriate type.  */
    6055              :           tree scalar = gimple_build (&seq, as_combined_fn (reduc_fn),
    6056              :                                       TREE_TYPE (vectype), vec);
    6057              :           scalar = gimple_convert (&seq, scalar_type, scalar);
    6058              :           scalar_results.safe_push (scalar);
    6059              :         }
    6060              :       gsi_insert_seq_before (&exit_gsi, seq, GSI_SAME_STMT);
    6061              :     }
    6062              :   else
    6063              :     {
    6064         1619 :       bool reduce_with_shift;
    6065         1619 :       tree vec_temp;
    6066              : 
    6067         1619 :       gcc_assert (slp_reduc || reduc_inputs.length () == 1);
    6068              : 
    6069              :       /* See if the target wants to do the final (shift) reduction
    6070              :          in a vector mode of smaller size and first reduce upper/lower
    6071              :          halves against each other.  */
    6072         1817 :       enum machine_mode mode1 = mode;
    6073         1817 :       tree stype = TREE_TYPE (vectype);
    6074         1817 :       if (compute_vectype != vectype)
    6075              :         {
    6076          459 :           stype = unsigned_type_for (stype);
    6077          459 :           gimple_seq stmts = NULL;
    6078          976 :           for (unsigned i = 0; i < reduc_inputs.length (); ++i)
    6079              :             {
    6080          517 :               tree new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR,
    6081          517 :                                             compute_vectype, reduc_inputs[i]);
    6082          517 :               reduc_inputs[i] = new_temp;
    6083              :             }
    6084          459 :           gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    6085              :         }
    6086         1817 :       unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
    6087         1817 :       unsigned nunits1 = nunits;
    6088         1817 :       if ((mode1 = targetm.vectorize.split_reduction (mode)) != mode
    6089         1817 :           && reduc_inputs.length () == 1)
    6090              :         {
    6091           41 :           nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
    6092              :           /* For SLP reductions we have to make sure lanes match up, but
    6093              :              since we're doing individual element final reduction reducing
    6094              :              vector width here is even more important.
    6095              :              ???  We can also separate lanes with permutes, for the common
    6096              :              case of power-of-two group-size odd/even extracts would work.  */
    6097           41 :           if (slp_reduc && nunits != nunits1)
    6098              :             {
    6099           41 :               nunits1 = least_common_multiple (nunits1, group_size);
    6100           82 :               gcc_assert (exact_log2 (nunits1) != -1 && nunits1 <= nunits);
    6101              :             }
    6102              :         }
    6103         1776 :       else if (!slp_reduc
    6104         1776 :                && (mode1 = targetm.vectorize.split_reduction (mode)) != mode)
    6105            0 :         nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
    6106              : 
    6107         1817 :       tree vectype1 = compute_vectype;
    6108         1817 :       if (mode1 != mode)
    6109              :         {
    6110           47 :           vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
    6111           47 :                                                           stype, nunits1);
    6112              :           /* First reduce the vector to the desired vector size we should
    6113              :              do shift reduction on by combining upper and lower halves.  */
    6114           47 :           gimple_seq stmts = NULL;
    6115           47 :           new_temp = vect_create_partial_epilog (reduc_inputs[0], vectype1,
    6116              :                                                  code, &stmts);
    6117           47 :           gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    6118           47 :           reduc_inputs[0] = new_temp;
    6119              :         }
    6120              : 
    6121         1817 :       reduce_with_shift = have_whole_vector_shift (mode1);
    6122          754 :       if (!VECTOR_MODE_P (mode1)
    6123         2569 :           || !directly_supported_p (code, vectype1))
    6124              :         reduce_with_shift = false;
    6125              : 
    6126         1800 :       if (reduce_with_shift && (!slp_reduc || group_size == 1))
    6127              :         {
    6128         1549 :           int element_bitsize = vector_element_bits (vectype1);
    6129              :           /* Enforced by vectorizable_reduction, which disallows SLP reductions
    6130              :              for variable-length vectors and also requires direct target support
    6131              :              for loop reductions.  */
    6132         1549 :           int nelements = TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
    6133         1549 :           vec_perm_builder sel;
    6134         1549 :           vec_perm_indices indices;
    6135              : 
    6136         1549 :           int elt_offset;
    6137              : 
    6138         1549 :           tree zero_vec = build_zero_cst (vectype1);
    6139              :           /* Case 2: Create:
    6140              :              for (offset = nelements/2; offset >= 1; offset/=2)
    6141              :                 {
    6142              :                   Create:  va' = vec_shift <va, offset>
    6143              :                   Create:  va = vop <va, va'>
    6144              :                 }  */
    6145              : 
    6146         1549 :           if (dump_enabled_p ())
    6147          384 :             dump_printf_loc (MSG_NOTE, vect_location,
    6148              :                              "Reduce using vector shifts\n");
    6149              : 
    6150         1549 :           gimple_seq stmts = NULL;
    6151         1549 :           new_temp = gimple_convert (&stmts, vectype1, reduc_inputs[0]);
    6152         1549 :           for (elt_offset = nelements / 2;
    6153         3401 :                elt_offset >= 1;
    6154         1852 :                elt_offset /= 2)
    6155              :             {
    6156         1852 :               calc_vec_perm_mask_for_shift (elt_offset, nelements, &sel);
    6157         1852 :               indices.new_vector (sel, 2, nelements);
    6158         1852 :               tree mask = vect_gen_perm_mask_any (vectype1, indices);
    6159         1852 :               new_name = gimple_build (&stmts, VEC_PERM_EXPR, vectype1,
    6160              :                                        new_temp, zero_vec, mask);
    6161         1852 :               new_temp = gimple_build (&stmts, code,
    6162              :                                        vectype1, new_name, new_temp);
    6163              :             }
    6164              : 
    6165              :           /* 2.4  Extract the final scalar result.  Create:
    6166              :              s_out3 = extract_field <v_out2, bitpos>  */
    6167              : 
    6168         1549 :           if (dump_enabled_p ())
    6169          384 :             dump_printf_loc (MSG_NOTE, vect_location,
    6170              :                              "extract scalar result\n");
    6171              : 
    6172         1549 :           new_temp = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype1),
    6173         1549 :                                    new_temp, bitsize_int (element_bitsize),
    6174         1549 :                                    bitsize_zero_node);
    6175         1549 :           new_temp = gimple_convert (&stmts, scalar_type, new_temp);
    6176         1549 :           gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    6177         1549 :           scalar_results.safe_push (new_temp);
    6178         1549 :         }
    6179              :       else
    6180              :         {
    6181              :           /* Case 3: Create:
    6182              :              s = extract_field <v_out2, 0>
    6183              :              for (offset = element_size;
    6184              :                   offset < vector_size;
    6185              :                   offset += element_size;)
    6186              :                {
    6187              :                  Create:  s' = extract_field <v_out2, offset>
    6188              :                  Create:  s = op <s, s'>  // For non SLP cases
    6189              :                }  */
    6190              : 
    6191          268 :           if (dump_enabled_p ())
    6192          143 :             dump_printf_loc (MSG_NOTE, vect_location,
    6193              :                              "Reduce using scalar code.\n");
    6194              : 
    6195          268 :           tree compute_type = TREE_TYPE (vectype1);
    6196          268 :           unsigned element_bitsize = vector_element_bits (vectype1);
    6197          268 :           unsigned vec_size_in_bits = element_bitsize
    6198          268 :             * TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
    6199          268 :           tree bitsize = bitsize_int (element_bitsize);
    6200          268 :           gimple_seq stmts = NULL;
    6201          691 :           FOR_EACH_VEC_ELT (reduc_inputs, i, vec_temp)
    6202              :             {
    6203          423 :               unsigned bit_offset;
    6204          846 :               new_temp = gimple_build (&stmts, BIT_FIELD_REF, compute_type,
    6205          423 :                                        vec_temp, bitsize, bitsize_zero_node);
    6206              : 
    6207              :               /* In SLP we don't need to apply reduction operation, so we just
    6208              :                  collect s' values in SCALAR_RESULTS.  */
    6209          423 :               if (slp_reduc)
    6210          413 :                 scalar_results.safe_push (new_temp);
    6211              : 
    6212          423 :               for (bit_offset = element_bitsize;
    6213         1422 :                    bit_offset < vec_size_in_bits;
    6214          999 :                    bit_offset += element_bitsize)
    6215              :                 {
    6216          999 :                   tree bitpos = bitsize_int (bit_offset);
    6217          999 :                   new_name = gimple_build (&stmts, BIT_FIELD_REF,
    6218              :                                            compute_type, vec_temp,
    6219              :                                            bitsize, bitpos);
    6220          999 :                   if (slp_reduc)
    6221              :                     {
    6222              :                       /* In SLP we don't need to apply reduction operation, so
    6223              :                          we just collect s' values in SCALAR_RESULTS.  */
    6224          989 :                       new_temp = new_name;
    6225          989 :                       scalar_results.safe_push (new_name);
    6226              :                     }
    6227              :                   else
    6228           10 :                     new_temp = gimple_build (&stmts, code, compute_type,
    6229              :                                              new_name, new_temp);
    6230              :                 }
    6231              :             }
    6232              : 
    6233              :           /* The only case where we need to reduce scalar results in a SLP
    6234              :              reduction, is unrolling.  If the size of SCALAR_RESULTS is
    6235              :              greater than GROUP_SIZE, we reduce them combining elements modulo
    6236              :              GROUP_SIZE.  */
    6237          268 :           if (slp_reduc)
    6238              :             {
    6239          258 :               tree res, first_res, new_res;
    6240              : 
    6241              :               /* Reduce multiple scalar results in case of SLP unrolling.  */
    6242          920 :               for (j = group_size; scalar_results.iterate (j, &res);
    6243              :                    j++)
    6244              :                 {
    6245          662 :                   first_res = scalar_results[j % group_size];
    6246          662 :                   new_res = gimple_build (&stmts, code, compute_type,
    6247              :                                           first_res, res);
    6248          662 :                   scalar_results[j % group_size] = new_res;
    6249              :                 }
    6250          258 :               scalar_results.truncate (group_size);
    6251         1256 :               for (k = 0; k < group_size; k++)
    6252         1480 :                 scalar_results[k] = gimple_convert (&stmts, scalar_type,
    6253          740 :                                                     scalar_results[k]);
    6254              :             }
    6255              :           else
    6256              :             {
    6257              :               /* Reduction chain - we have one scalar to keep in
    6258              :                  SCALAR_RESULTS.  */
    6259           10 :               new_temp = gimple_convert (&stmts, scalar_type, new_temp);
    6260           10 :               scalar_results.safe_push (new_temp);
    6261              :             }
    6262              : 
    6263          268 :           gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    6264              :         }
    6265              : 
    6266         1817 :       if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
    6267            0 :           && induc_val)
    6268              :         {
    6269              :           /* Earlier we set the initial value to be a vector if induc_val
    6270              :              values.  Check the result and if it is induc_val then replace
    6271              :              with the original initial value, unless induc_val is
    6272              :              the same as initial_def already.  */
    6273            0 :           tree zcompare = make_ssa_name (boolean_type_node);
    6274            0 :           epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
    6275            0 :                                              scalar_results[0], induc_val);
    6276            0 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    6277            0 :           tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
    6278            0 :           tree tmp = make_ssa_name (new_scalar_dest);
    6279            0 :           epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
    6280            0 :                                              initial_def, scalar_results[0]);
    6281            0 :           gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
    6282            0 :           scalar_results[0] = tmp;
    6283              :         }
    6284              :     }
    6285              : 
    6286              :   /* 2.5 Adjust the final result by the initial value of the reduction
    6287              :          variable. (When such adjustment is not needed, then
    6288              :          'adjustment_def' is zero).  For example, if code is PLUS we create:
    6289              :          new_temp = loop_exit_def + adjustment_def  */
    6290              : 
    6291        22094 :   if (adjustment_def)
    6292              :     {
    6293        15570 :       gcc_assert (!slp_reduc || group_size == 1);
    6294        15570 :       gimple_seq stmts = NULL;
    6295        15570 :       if (double_reduc)
    6296              :         {
    6297            0 :           gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def)));
    6298            0 :           adjustment_def = gimple_convert (&stmts, vectype, adjustment_def);
    6299            0 :           new_temp = gimple_build (&stmts, code, vectype,
    6300            0 :                                    reduc_inputs[0], adjustment_def);
    6301              :         }
    6302              :       else
    6303              :         {
    6304        15570 :           new_temp = scalar_results[0];
    6305        15570 :           gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
    6306        15570 :           adjustment_def = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
    6307              :                                            adjustment_def);
    6308        15570 :           new_temp = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
    6309              :                                      new_temp);
    6310        15570 :           new_temp = gimple_build (&stmts, code, TREE_TYPE (compute_vectype),
    6311              :                                    new_temp, adjustment_def);
    6312        15570 :           new_temp = gimple_convert (&stmts, scalar_type, new_temp);
    6313              :         }
    6314              : 
    6315        15570 :       epilog_stmt = gimple_seq_last_stmt (stmts);
    6316        15570 :       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
    6317        15570 :       scalar_results[0] = new_temp;
    6318              :     }
    6319              : 
    6320              :   /* Record this operation if it could be reused by the epilogue loop.  */
    6321        22094 :   if (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION
    6322        22094 :       && reduc_inputs.length () == 1)
    6323        21880 :     loop_vinfo->reusable_accumulators.put (scalar_results[0],
    6324              :                                            { orig_reduc_input, reduc_info });
    6325              : 
    6326              :   /* 2.6  Handle the loop-exit phis.  Replace the uses of scalar loop-exit
    6327              :           phis with new adjusted scalar results, i.e., replace use <s_out0>
    6328              :           with use <s_out4>.
    6329              : 
    6330              :      Transform:
    6331              :         loop_exit:
    6332              :           s_out0 = phi <s_loop>                 # (scalar) EXIT_PHI
    6333              :           v_out1 = phi <VECT_DEF>               # NEW_EXIT_PHI
    6334              :           v_out2 = reduce <v_out1>
    6335              :           s_out3 = extract_field <v_out2, 0>
    6336              :           s_out4 = adjust_result <s_out3>
    6337              :           use <s_out0>
    6338              :           use <s_out0>
    6339              : 
    6340              :      into:
    6341              : 
    6342              :         loop_exit:
    6343              :           s_out0 = phi <s_loop>                 # (scalar) EXIT_PHI
    6344              :           v_out1 = phi <VECT_DEF>               # NEW_EXIT_PHI
    6345              :           v_out2 = reduce <v_out1>
    6346              :           s_out3 = extract_field <v_out2, 0>
    6347              :           s_out4 = adjust_result <s_out3>
    6348              :           use <s_out4>
    6349              :           use <s_out4> */
    6350              : 
    6351        44188 :   gcc_assert (live_out_stmts.size () == scalar_results.length ());
    6352        22094 :   auto_vec<gimple *> phis;
    6353        44670 :   for (k = 0; k < live_out_stmts.size (); k++)
    6354              :     {
    6355        22576 :       stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
    6356        22576 :       tree scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
    6357              : 
    6358              :       /* Find the loop-closed-use at the loop exit of the original scalar
    6359              :          result.  (The reduction result is expected to have two immediate uses,
    6360              :          one at the latch block, and one at the loop exit).  Note with
    6361              :          early break we can have two exit blocks, so pick the correct PHI.  */
    6362        91521 :       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
    6363        68945 :         if (!is_gimple_debug (USE_STMT (use_p))
    6364        68945 :             && !flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
    6365              :           {
    6366        22569 :             gcc_assert (is_a <gphi *> (USE_STMT (use_p)));
    6367        22569 :             if (gimple_bb (USE_STMT (use_p)) == loop_exit->dest)
    6368        22561 :               phis.safe_push (USE_STMT (use_p));
    6369        22576 :           }
    6370              : 
    6371        45137 :       FOR_EACH_VEC_ELT (phis, i, exit_phi)
    6372              :         {
    6373              :           /* Replace the uses:  */
    6374        22561 :           orig_name = PHI_RESULT (exit_phi);
    6375              : 
    6376              :           /* Look for a single use at the target of the skip edge.  */
    6377        22561 :           if (unify_with_main_loop_p)
    6378              :             {
    6379           30 :               use_operand_p use_p;
    6380           30 :               gimple *user;
    6381           30 :               if (!single_imm_use (orig_name, &use_p, &user))
    6382            0 :                 gcc_unreachable ();
    6383           30 :               orig_name = gimple_get_lhs (user);
    6384              :             }
    6385              : 
    6386        22561 :           scalar_result = scalar_results[k];
    6387        60879 :           FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
    6388              :             {
    6389        38318 :               gphi *use_phi = dyn_cast <gphi *> (use_stmt);
    6390        76680 :               FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
    6391              :                 {
    6392        38340 :                   if (use_phi
    6393        38340 :                       && (phi_arg_edge_from_use (use_p)->flags & EDGE_ABNORMAL))
    6394              :                     {
    6395            0 :                       gcc_assert (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (orig_name));
    6396            0 :                       SSA_NAME_OCCURS_IN_ABNORMAL_PHI (scalar_result) = 1;
    6397              :                     }
    6398        38340 :                   SET_USE (use_p, scalar_result);
    6399              :                 }
    6400        38318 :               update_stmt (use_stmt);
    6401        22561 :             }
    6402              :         }
    6403              : 
    6404        22576 :       phis.truncate (0);
    6405              :     }
    6406        22094 : }
    6407              : 
    6408              : /* Return a vector of type VECTYPE that is equal to the vector select
    6409              :    operation "MASK ? VEC : IDENTITY".  Insert the select statements
    6410              :    before GSI.  */
    6411              : 
    6412              : static tree
    6413            9 : merge_with_identity (gimple_stmt_iterator *gsi, tree mask, tree vectype,
    6414              :                      tree vec, tree identity)
    6415              : {
    6416            9 :   tree cond = make_temp_ssa_name (vectype, NULL, "cond");
    6417            9 :   gimple *new_stmt = gimple_build_assign (cond, VEC_COND_EXPR,
    6418              :                                           mask, vec, identity);
    6419            9 :   gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    6420            9 :   return cond;
    6421              : }
    6422              : 
    6423              : /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
    6424              :    order, starting with LHS.  Insert the extraction statements before GSI and
    6425              :    associate the new scalar SSA names with variable SCALAR_DEST.
    6426              :    If MASK is nonzero mask the input and then operate on it unconditionally.
    6427              :    Return the SSA name for the result.  */
    6428              : 
    6429              : static tree
    6430         1309 : vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
    6431              :                        tree_code code, tree lhs, tree vector_rhs,
    6432              :                        tree mask)
    6433              : {
    6434         1309 :   tree vectype = TREE_TYPE (vector_rhs);
    6435         1309 :   tree scalar_type = TREE_TYPE (vectype);
    6436         1309 :   tree bitsize = TYPE_SIZE (scalar_type);
    6437         1309 :   unsigned HOST_WIDE_INT vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
    6438         1309 :   unsigned HOST_WIDE_INT element_bitsize = tree_to_uhwi (bitsize);
    6439              : 
    6440              :   /* Re-create a VEC_COND_EXPR to mask the input here in order to be able
    6441              :      to perform an unconditional element-wise reduction of it.  */
    6442         1309 :   if (mask)
    6443              :     {
    6444           76 :       tree masked_vector_rhs = make_temp_ssa_name (vectype, NULL,
    6445              :                                                    "masked_vector_rhs");
    6446           76 :       tree neutral_op = neutral_op_for_reduction (scalar_type, code, NULL_TREE,
    6447              :                                                   false);
    6448           76 :       tree vector_identity = build_vector_from_val (vectype, neutral_op);
    6449           76 :       gassign *select = gimple_build_assign (masked_vector_rhs, VEC_COND_EXPR,
    6450              :                                              mask, vector_rhs, vector_identity);
    6451           76 :       gsi_insert_before (gsi, select, GSI_SAME_STMT);
    6452           76 :       vector_rhs = masked_vector_rhs;
    6453              :     }
    6454              : 
    6455         1309 :   for (unsigned HOST_WIDE_INT bit_offset = 0;
    6456         5419 :        bit_offset < vec_size_in_bits;
    6457         4110 :        bit_offset += element_bitsize)
    6458              :     {
    6459         4110 :       tree bitpos = bitsize_int (bit_offset);
    6460         4110 :       tree rhs = build3 (BIT_FIELD_REF, scalar_type, vector_rhs,
    6461              :                          bitsize, bitpos);
    6462              : 
    6463         4110 :       gassign *stmt = gimple_build_assign (scalar_dest, rhs);
    6464         4110 :       rhs = make_ssa_name (scalar_dest, stmt);
    6465         4110 :       gimple_assign_set_lhs (stmt, rhs);
    6466         4110 :       gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
    6467              :       /* Fold the vector extract, combining it with a previous reversal
    6468              :          like seen in PR90579.  */
    6469         4110 :       auto gsi2 = gsi_for_stmt (stmt);
    6470         4110 :       if (fold_stmt (&gsi2, follow_all_ssa_edges))
    6471          450 :         update_stmt (gsi_stmt  (gsi2));
    6472              : 
    6473         4110 :       stmt = gimple_build_assign (scalar_dest, code, lhs, rhs);
    6474         4110 :       tree new_name = make_ssa_name (scalar_dest, stmt);
    6475         4110 :       gimple_assign_set_lhs (stmt, new_name);
    6476         4110 :       gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
    6477         4110 :       lhs = new_name;
    6478              :     }
    6479         1309 :   return lhs;
    6480              : }
    6481              : 
    6482              : /* Get a masked internal function equivalent to REDUC_FN.  VECTYPE_IN is the
    6483              :    type of the vector input.  */
    6484              : 
    6485              : static internal_fn
    6486         3191 : get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
    6487              : {
    6488         3191 :   internal_fn mask_reduc_fn;
    6489         3191 :   internal_fn mask_len_reduc_fn;
    6490              : 
    6491         3191 :   switch (reduc_fn)
    6492              :     {
    6493            0 :     case IFN_FOLD_LEFT_PLUS:
    6494            0 :       mask_reduc_fn = IFN_MASK_FOLD_LEFT_PLUS;
    6495            0 :       mask_len_reduc_fn = IFN_MASK_LEN_FOLD_LEFT_PLUS;
    6496            0 :       break;
    6497              : 
    6498              :     default:
    6499              :       return IFN_LAST;
    6500              :     }
    6501              : 
    6502            0 :   if (direct_internal_fn_supported_p (mask_reduc_fn, vectype_in,
    6503              :                                       OPTIMIZE_FOR_SPEED))
    6504              :     return mask_reduc_fn;
    6505            0 :   if (direct_internal_fn_supported_p (mask_len_reduc_fn, vectype_in,
    6506              :                                       OPTIMIZE_FOR_SPEED))
    6507            0 :     return mask_len_reduc_fn;
    6508              :   return IFN_LAST;
    6509              : }
    6510              : 
    6511              : /* Perform an in-order reduction (FOLD_LEFT_REDUCTION).  STMT_INFO is the
    6512              :    statement that sets the live-out value.  REDUC_DEF_STMT is the phi
    6513              :    statement.  CODE is the operation performed by STMT_INFO and OPS are
    6514              :    its scalar operands.  REDUC_INDEX is the index of the operand in
    6515              :    OPS that is set by REDUC_DEF_STMT.  REDUC_FN is the function that
    6516              :    implements in-order reduction, or IFN_LAST if we should open-code it.
    6517              :    VECTYPE_IN is the type of the vector input.  MASKS specifies the masks
    6518              :    that should be used to control the operation in a fully-masked loop.  */
    6519              : 
    6520              : static bool
    6521          977 : vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
    6522              :                                stmt_vec_info stmt_info,
    6523              :                                gimple_stmt_iterator *gsi,
    6524              :                                slp_tree slp_node,
    6525              :                                code_helper code, internal_fn reduc_fn,
    6526              :                                int num_ops, tree vectype_in,
    6527              :                                int reduc_index, vec_loop_masks *masks,
    6528              :                                vec_loop_lens *lens)
    6529              : {
    6530          977 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    6531          977 :   tree vectype_out = SLP_TREE_VECTYPE (slp_node);
    6532          977 :   internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
    6533              : 
    6534          977 :   gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
    6535              : 
    6536          977 :   bool is_cond_op = false;
    6537          977 :   if (!code.is_tree_code ())
    6538              :     {
    6539           29 :       code = conditional_internal_fn_code (internal_fn (code));
    6540           29 :       gcc_assert (code != ERROR_MARK);
    6541              :       is_cond_op = true;
    6542              :     }
    6543              : 
    6544          977 :   gcc_assert (TREE_CODE_LENGTH (tree_code (code)) == binary_op);
    6545              : 
    6546          977 :   gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out),
    6547              :                         TYPE_VECTOR_SUBPARTS (vectype_in)));
    6548              : 
    6549              :   /* ???  We should, when transforming the cycle PHI, record the existing
    6550              :      scalar def as vector def so looking up the vector def works.  This
    6551              :      would also allow generalizing this for reduction paths of length > 1
    6552              :      and/or SLP reductions.  */
    6553          977 :   slp_tree reduc_node = SLP_TREE_CHILDREN (slp_node)[reduc_index];
    6554          977 :   stmt_vec_info reduc_var_def = SLP_TREE_SCALAR_STMTS (reduc_node)[0];
    6555          977 :   tree reduc_var = gimple_get_lhs (STMT_VINFO_STMT (reduc_var_def));
    6556              : 
    6557              :   /* The operands either come from a binary operation or an IFN_COND operation.
    6558              :      The former is a gimple assign with binary rhs and the latter is a
    6559              :      gimple call with four arguments.  */
    6560          977 :   gcc_assert (num_ops == 2 || num_ops == 4);
    6561              : 
    6562          977 :   auto_vec<tree> vec_oprnds0, vec_opmask;
    6563          977 :   vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[(is_cond_op ? 2 : 0)
    6564          977 :                                                   + (1 - reduc_index)],
    6565              :                                                   &vec_oprnds0);
    6566              :   /* For an IFN_COND_OP we also need the vector mask operand.  */
    6567          977 :   if (is_cond_op)
    6568           29 :     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], &vec_opmask);
    6569              : 
    6570              :   /* The transform below relies on preserving the original scalar PHI
    6571              :      and its latch def which we replace.  So work backwards from there.  */
    6572          977 :   tree scalar_dest
    6573          977 :     = gimple_phi_arg_def_from_edge (as_a <gphi *> (STMT_VINFO_STMT
    6574              :                                                      (reduc_var_def)),
    6575          977 :                                     loop_latch_edge (loop));
    6576          977 :   stmt_vec_info scalar_dest_def_info
    6577          977 :     = vect_stmt_to_vectorize (loop_vinfo->lookup_def (scalar_dest));
    6578          977 :   tree scalar_type = TREE_TYPE (scalar_dest);
    6579              : 
    6580          977 :   int vec_num = vec_oprnds0.length ();
    6581          977 :   tree vec_elem_type = TREE_TYPE (vectype_out);
    6582          977 :   gcc_checking_assert (useless_type_conversion_p (scalar_type, vec_elem_type));
    6583              : 
    6584          977 :   tree vector_identity = NULL_TREE;
    6585          977 :   if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    6586              :     {
    6587            2 :       vector_identity = build_zero_cst (vectype_out);
    6588            2 :       if (!HONOR_SIGNED_ZEROS (vectype_out))
    6589              :         ;
    6590              :       else
    6591              :         {
    6592            2 :           gcc_assert (!HONOR_SIGN_DEPENDENT_ROUNDING (vectype_out));
    6593            2 :           vector_identity = const_unop (NEGATE_EXPR, vectype_out,
    6594              :                                         vector_identity);
    6595              :         }
    6596              :     }
    6597              : 
    6598          977 :   tree scalar_dest_var = vect_create_destination_var (scalar_dest, NULL);
    6599          977 :   int i;
    6600          977 :   tree def0;
    6601         3263 :   FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
    6602              :     {
    6603         1309 :       gimple *new_stmt;
    6604         1309 :       tree mask = NULL_TREE;
    6605         1309 :       tree len = NULL_TREE;
    6606         1309 :       tree bias = NULL_TREE;
    6607         1309 :       if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
    6608              :         {
    6609            9 :           tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    6610              :                                                vec_num, vectype_in, i);
    6611            9 :           if (is_cond_op)
    6612            9 :             mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (loop_mask),
    6613            9 :                                      loop_mask, vec_opmask[i], gsi);
    6614              :           else
    6615              :             mask = loop_mask;
    6616              :         }
    6617         1300 :       else if (is_cond_op)
    6618           67 :         mask = vec_opmask[i];
    6619         1309 :       if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
    6620              :         {
    6621            0 :           len = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num, vectype_in,
    6622              :                                    i, 1, false);
    6623            0 :           signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
    6624            0 :           bias = build_int_cst (intQI_type_node, biasval);
    6625            0 :           if (!is_cond_op)
    6626            0 :             mask = build_minus_one_cst (truth_type_for (vectype_in));
    6627              :         }
    6628              : 
    6629              :       /* Handle MINUS by adding the negative.  */
    6630         1309 :       if (reduc_fn != IFN_LAST && code == MINUS_EXPR)
    6631              :         {
    6632            0 :           tree negated = make_ssa_name (vectype_out);
    6633            0 :           new_stmt = gimple_build_assign (negated, NEGATE_EXPR, def0);
    6634            0 :           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    6635            0 :           def0 = negated;
    6636              :         }
    6637              : 
    6638            9 :       if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
    6639         1318 :           && mask && mask_reduc_fn == IFN_LAST)
    6640            9 :         def0 = merge_with_identity (gsi, mask, vectype_out, def0,
    6641              :                                     vector_identity);
    6642              : 
    6643              :       /* On the first iteration the input is simply the scalar phi
    6644              :          result, and for subsequent iterations it is the output of
    6645              :          the preceding operation.  */
    6646         1309 :       if (reduc_fn != IFN_LAST || (mask && mask_reduc_fn != IFN_LAST))
    6647              :         {
    6648            0 :           if (mask && len && mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
    6649            0 :             new_stmt = gimple_build_call_internal (mask_reduc_fn, 5, reduc_var,
    6650              :                                                    def0, mask, len, bias);
    6651            0 :           else if (mask && mask_reduc_fn == IFN_MASK_FOLD_LEFT_PLUS)
    6652            0 :             new_stmt = gimple_build_call_internal (mask_reduc_fn, 3, reduc_var,
    6653              :                                                    def0, mask);
    6654              :           else
    6655            0 :             new_stmt = gimple_build_call_internal (reduc_fn, 2, reduc_var,
    6656              :                                                    def0);
    6657              :           /* For chained SLP reductions the output of the previous reduction
    6658              :              operation serves as the input of the next. For the final statement
    6659              :              the output cannot be a temporary - we reuse the original
    6660              :              scalar destination of the last statement.  */
    6661            0 :           if (i != vec_num - 1)
    6662              :             {
    6663            0 :               gimple_set_lhs (new_stmt, scalar_dest_var);
    6664            0 :               reduc_var = make_ssa_name (scalar_dest_var, new_stmt);
    6665            0 :               gimple_set_lhs (new_stmt, reduc_var);
    6666              :             }
    6667              :         }
    6668              :       else
    6669              :         {
    6670         1309 :           reduc_var = vect_expand_fold_left (gsi, scalar_dest_var,
    6671              :                                              tree_code (code), reduc_var, def0,
    6672              :                                              mask);
    6673         1309 :           new_stmt = SSA_NAME_DEF_STMT (reduc_var);
    6674              :           /* Remove the statement, so that we can use the same code paths
    6675              :              as for statements that we've just created.  */
    6676         1309 :           gimple_stmt_iterator tmp_gsi = gsi_for_stmt (new_stmt);
    6677         1309 :           gsi_remove (&tmp_gsi, true);
    6678              :         }
    6679              : 
    6680         1309 :       if (i == vec_num - 1)
    6681              :         {
    6682          977 :           gimple_set_lhs (new_stmt, scalar_dest);
    6683          977 :           vect_finish_replace_stmt (loop_vinfo,
    6684              :                                     scalar_dest_def_info,
    6685              :                                     new_stmt);
    6686              :         }
    6687              :       else
    6688          332 :         vect_finish_stmt_generation (loop_vinfo,
    6689              :                                      scalar_dest_def_info,
    6690              :                                      new_stmt, gsi);
    6691              : 
    6692         1309 :       slp_node->push_vec_def (new_stmt);
    6693              :     }
    6694              : 
    6695          977 :   return true;
    6696          977 : }
    6697              : 
    6698              : /* Function is_nonwrapping_integer_induction.
    6699              : 
    6700              :    Check if STMT_VINFO (which is part of loop LOOP) both increments and
    6701              :    does not cause overflow.  */
    6702              : 
    6703              : static bool
    6704          414 : is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, class loop *loop)
    6705              : {
    6706          414 :   gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
    6707          414 :   tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo);
    6708          414 :   tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo);
    6709          414 :   tree lhs_type = TREE_TYPE (gimple_phi_result (phi));
    6710          414 :   widest_int ni, max_loop_value, lhs_max;
    6711          414 :   wi::overflow_type overflow = wi::OVF_NONE;
    6712              : 
    6713              :   /* Make sure this is a regular induction.  */
    6714          414 :   if (STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add)
    6715              :     return false;
    6716              : 
    6717              :   /* Make sure the loop is integer based.  */
    6718          408 :   if (TREE_CODE (base) != INTEGER_CST
    6719          109 :       || TREE_CODE (step) != INTEGER_CST)
    6720              :     return false;
    6721              : 
    6722              :   /* Check that the max size of the loop will not wrap.  */
    6723              : 
    6724          109 :   if (TYPE_OVERFLOW_UNDEFINED (lhs_type))
    6725              :     return true;
    6726              : 
    6727            8 :   if (! max_stmt_executions (loop, &ni))
    6728              :     return false;
    6729              : 
    6730            8 :   max_loop_value = wi::mul (wi::to_widest (step), ni, TYPE_SIGN (lhs_type),
    6731            8 :                             &overflow);
    6732            8 :   if (overflow)
    6733              :     return false;
    6734              : 
    6735            8 :   max_loop_value = wi::add (wi::to_widest (base), max_loop_value,
    6736           16 :                             TYPE_SIGN (lhs_type), &overflow);
    6737            8 :   if (overflow)
    6738              :     return false;
    6739              : 
    6740            8 :   return (wi::min_precision (max_loop_value, TYPE_SIGN (lhs_type))
    6741            8 :           <= TYPE_PRECISION (lhs_type));
    6742          414 : }
    6743              : 
    6744              : /* Check if masking can be supported by inserting a conditional expression.
    6745              :    CODE is the code for the operation.  COND_FN is the conditional internal
    6746              :    function, if it exists.  VECTYPE_IN is the type of the vector input.  */
    6747              : static bool
    6748         6103 : use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
    6749              :                          tree vectype_in)
    6750              : {
    6751         6103 :   if (cond_fn != IFN_LAST
    6752         6103 :       && direct_internal_fn_supported_p (cond_fn, vectype_in,
    6753              :                                          OPTIMIZE_FOR_SPEED))
    6754              :     return false;
    6755              : 
    6756         4376 :   if (code.is_tree_code ())
    6757         4356 :     switch (tree_code (code))
    6758              :       {
    6759          397 :       case DOT_PROD_EXPR:
    6760          397 :       case SAD_EXPR:
    6761          397 :         return true;
    6762              : 
    6763              :       default:
    6764              :         break;
    6765              :       }
    6766              :   return false;
    6767              : }
    6768              : 
    6769              : /* Insert a conditional expression to enable masked vectorization.  CODE is the
    6770              :    code for the operation.  VOP is the array of operands.  MASK is the loop
    6771              :    mask.  GSI is a statement iterator used to place the new conditional
    6772              :    expression.  */
    6773              : static void
    6774            4 : build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
    6775              :                       gimple_stmt_iterator *gsi)
    6776              : {
    6777            4 :   switch (tree_code (code))
    6778              :     {
    6779            4 :     case DOT_PROD_EXPR:
    6780            4 :       {
    6781            4 :         tree vectype = TREE_TYPE (vop[1]);
    6782            4 :         tree zero = build_zero_cst (vectype);
    6783            4 :         tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
    6784            4 :         gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
    6785              :                                                mask, vop[1], zero);
    6786            4 :         gsi_insert_before (gsi, select, GSI_SAME_STMT);
    6787            4 :         vop[1] = masked_op1;
    6788            4 :         break;
    6789              :       }
    6790              : 
    6791            0 :     case SAD_EXPR:
    6792            0 :       {
    6793            0 :         tree vectype = TREE_TYPE (vop[1]);
    6794            0 :         tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
    6795            0 :         gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
    6796              :                                                mask, vop[1], vop[0]);
    6797            0 :         gsi_insert_before (gsi, select, GSI_SAME_STMT);
    6798            0 :         vop[1] = masked_op1;
    6799            0 :         break;
    6800              :       }
    6801              : 
    6802            0 :     default:
    6803            0 :       gcc_unreachable ();
    6804              :     }
    6805            4 : }
    6806              : 
    6807              : /* Given an operation with CODE in loop reduction path whose reduction PHI is
    6808              :    specified by REDUC_INFO, the operation has TYPE of scalar result, and its
    6809              :    input vectype is represented by VECTYPE_IN. The vectype of vectorized result
    6810              :    may be different from VECTYPE_IN, either in base type or vectype lanes,
    6811              :    lane-reducing operation is the case.  This function check if it is possible,
    6812              :    and how to perform partial vectorization on the operation in the context
    6813              :    of LOOP_VINFO.  It returns whether partial vectors are possible.  The
    6814              :    caller is responsible to register the usage appropriately.  */
    6815              : 
    6816              : static bool
    6817         4293 : vect_reduction_update_partial_vector_usage (loop_vec_info loop_vinfo,
    6818              :                                             vect_reduc_info reduc_info,
    6819              :                                             code_helper code, tree type,
    6820              :                                             tree vectype_in)
    6821              : {
    6822         4293 :   enum vect_reduction_type reduc_type = VECT_REDUC_INFO_TYPE (reduc_info);
    6823         4293 :   internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
    6824         4293 :   internal_fn cond_fn
    6825         1157 :     = ((code.is_internal_fn ()
    6826         1157 :         && internal_fn_mask_index ((internal_fn)code) != -1)
    6827         4293 :        ? (internal_fn)code : get_conditional_internal_fn (code, type));
    6828              : 
    6829         4293 :   if (reduc_type != FOLD_LEFT_REDUCTION
    6830         3411 :       && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
    6831         7588 :       && (cond_fn == IFN_LAST
    6832         3295 :           || !direct_internal_fn_supported_p (cond_fn, vectype_in,
    6833              :                                               OPTIMIZE_FOR_SPEED)))
    6834              :     {
    6835         2079 :       if (dump_enabled_p ())
    6836          101 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6837              :                          "can't operate on partial vectors because"
    6838              :                          " no conditional operation is available.\n");
    6839         2079 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    6840              :     }
    6841         2214 :   else if (reduc_type == FOLD_LEFT_REDUCTION
    6842         2214 :            && reduc_fn == IFN_LAST
    6843         2214 :            && !expand_vec_cond_expr_p (vectype_in, truth_type_for (vectype_in)))
    6844              :     {
    6845            0 :       if (dump_enabled_p ())
    6846            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6847              :                         "can't operate on partial vectors because"
    6848              :                         " no conditional operation is available.\n");
    6849            0 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    6850              :     }
    6851         2214 :   else if (reduc_type == FOLD_LEFT_REDUCTION
    6852          882 :            && internal_fn_mask_index (reduc_fn) == -1
    6853          882 :            && FLOAT_TYPE_P (vectype_in)
    6854         3096 :            && HONOR_SIGN_DEPENDENT_ROUNDING (vectype_in))
    6855              :     {
    6856            0 :       if (dump_enabled_p ())
    6857            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6858              :                          "can't operate on partial vectors because"
    6859              :                          " signed zeros cannot be preserved.\n");
    6860            0 :       LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
    6861              :     }
    6862              :   else
    6863              :     return true;
    6864              :   return false;
    6865              : }
    6866              : 
    6867              : /* Check if STMT_INFO is a lane-reducing operation that can be vectorized in
    6868              :    the context of LOOP_VINFO, and vector cost will be recorded in COST_VEC,
    6869              :    and the analysis is for slp if SLP_NODE is not NULL.
    6870              : 
    6871              :    For a lane-reducing operation, the loop reduction path that it lies in,
    6872              :    may contain normal operation, or other lane-reducing operation of different
    6873              :    input type size, an example as:
    6874              : 
    6875              :      int sum = 0;
    6876              :      for (i)
    6877              :        {
    6878              :          ...
    6879              :          sum += d0[i] * d1[i];       // dot-prod <vector(16) char>
    6880              :          sum += w[i];                // widen-sum <vector(16) char>
    6881              :          sum += abs(s0[i] - s1[i]);  // sad <vector(8) short>
    6882              :          sum += n[i];                // normal <vector(4) int>
    6883              :          ...
    6884              :        }
    6885              : 
    6886              :    Vectorization factor is essentially determined by operation whose input
    6887              :    vectype has the most lanes ("vector(16) char" in the example), while we
    6888              :    need to choose input vectype with the least lanes ("vector(4) int" in the
    6889              :    example) to determine effective number of vector reduction PHIs.  */
    6890              : 
    6891              : bool
    6892       415227 : vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
    6893              :                             slp_tree slp_node, stmt_vector_for_cost *cost_vec)
    6894              : {
    6895       415227 :   gimple *stmt = stmt_info->stmt;
    6896              : 
    6897       415227 :   if (!lane_reducing_stmt_p (stmt))
    6898              :     return false;
    6899              : 
    6900          740 :   tree type = TREE_TYPE (gimple_assign_lhs (stmt));
    6901              : 
    6902          740 :   if (!INTEGRAL_TYPE_P (type) && !SCALAR_FLOAT_TYPE_P (type))
    6903              :     return false;
    6904              : 
    6905              :   /* Do not try to vectorize bit-precision reductions.  */
    6906          740 :   if (!type_has_mode_precision_p (type))
    6907              :     return false;
    6908              : 
    6909          740 :   vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
    6910              : 
    6911              :   /* TODO: Support lane-reducing operation that does not directly participate
    6912              :      in loop reduction.  */
    6913          740 :   if (!reduc_info)
    6914              :     return false;
    6915              : 
    6916              :   /* Lane-reducing pattern inside any inner loop of LOOP_VINFO is not
    6917              :      recognized.  */
    6918          740 :   gcc_assert (!nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info));
    6919          740 :   gcc_assert (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION);
    6920              : 
    6921         2960 :   for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
    6922              :     {
    6923         2220 :       slp_tree slp_op;
    6924         2220 :       tree op;
    6925         2220 :       tree vectype;
    6926         2220 :       enum vect_def_type dt;
    6927              : 
    6928         2220 :       if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
    6929              :                                &slp_op, &dt, &vectype))
    6930              :         {
    6931            0 :           if (dump_enabled_p ())
    6932            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6933              :                              "use not simple.\n");
    6934            0 :           return false;
    6935              :         }
    6936              : 
    6937         2220 :       if (!vectype)
    6938              :         {
    6939           15 :           vectype = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op),
    6940              :                                                  slp_op);
    6941           15 :           if (!vectype)
    6942              :             return false;
    6943              :         }
    6944              : 
    6945         2220 :       if (!vect_maybe_update_slp_op_vectype (slp_op, vectype))
    6946              :         {
    6947            0 :           if (dump_enabled_p ())
    6948            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6949              :                              "incompatible vector types for invariants\n");
    6950              :           return false;
    6951              :         }
    6952              : 
    6953         2220 :       if (i == STMT_VINFO_REDUC_IDX (stmt_info))
    6954          740 :         continue;
    6955              : 
    6956              :       /* There should be at most one cycle def in the stmt.  */
    6957         1480 :       if (VECTORIZABLE_CYCLE_DEF (dt))
    6958              :         return false;
    6959              :     }
    6960              : 
    6961          740 :   slp_tree node_in = SLP_TREE_CHILDREN (slp_node)[0];
    6962          740 :   tree vectype_in = SLP_TREE_VECTYPE (node_in);
    6963          740 :   gcc_assert (vectype_in);
    6964              : 
    6965              :   /* Compute number of effective vector statements for costing from the
    6966              :      number of input lanes allow for excess lanes in the last input vector.  */
    6967          740 :   unsigned int ncopies_for_cost, excess_elts;
    6968          740 :   if (!vect_get_num_copies_for_invariant (loop_vinfo, node_in,
    6969              :                                           &ncopies_for_cost,
    6970              :                                           &excess_elts))
    6971              :     {
    6972            0 :       if (dump_enabled_p ())
    6973            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6974              :                          "incompatible vector types for invariants\n");
    6975              :       return false;
    6976              :     }
    6977          740 :   gcc_assert (ncopies_for_cost >= 1);
    6978              : 
    6979          740 :   if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
    6980              :     {
    6981          116 :       enum tree_code code = gimple_assign_rhs_code (stmt);
    6982          116 :       if (vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
    6983          116 :                                                       code, type, vectype_in))
    6984              :         {
    6985          116 :           internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
    6986          116 :           internal_fn mask_reduc_fn
    6987          116 :             = get_masked_reduction_fn (reduc_fn, vectype_in);
    6988          116 :           vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    6989          116 :           vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
    6990          116 :           if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
    6991            0 :             vect_record_loop_len (loop_vinfo, lens, ncopies_for_cost,
    6992              :                                   vectype_in, 1);
    6993              :           else
    6994          116 :             vect_record_loop_mask (loop_vinfo, masks, ncopies_for_cost,
    6995              :                                    vectype_in, NULL);
    6996              :         }
    6997              :     }
    6998              : 
    6999          740 :   if (vect_is_emulated_mixed_dot_prod (slp_node))
    7000              :     {
    7001              :       /* We need extra two invariants: one that contains the minimum signed
    7002              :          value and one that contains half of its negative.  */
    7003           15 :       int prologue_stmts = 2;
    7004           15 :       unsigned cost = record_stmt_cost (cost_vec, prologue_stmts,
    7005              :                                         scalar_to_vec, slp_node, 0,
    7006              :                                         vect_prologue);
    7007           15 :       if (dump_enabled_p ())
    7008            0 :         dump_printf (MSG_NOTE, "vectorizable_lane_reducing: "
    7009              :                      "extra prologue_cost = %d .\n", cost);
    7010              : 
    7011              :       /* Three dot-products and a subtraction.  */
    7012           15 :       ncopies_for_cost *= 4;
    7013              :     }
    7014              : 
    7015          740 :   record_stmt_cost (cost_vec, (int) ncopies_for_cost, vector_stmt, slp_node,
    7016              :                     0, vect_body);
    7017              : 
    7018              :   /* Transform via vect_transform_reduction.  */
    7019          740 :   SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
    7020          740 :   return true;
    7021              : }
    7022              : 
    7023              : /* Function vectorizable_reduction.
    7024              : 
    7025              :    Check if STMT_INFO performs a reduction operation that can be vectorized.
    7026              :    If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
    7027              :    stmt to replace it, put it in VEC_STMT, and insert it at GSI.
    7028              :    Return true if STMT_INFO is vectorizable in this way.
    7029              : 
    7030              :    This function also handles reduction idioms (patterns) that have been
    7031              :    recognized in advance during vect_pattern_recog.  In this case, STMT_INFO
    7032              :    may be of this form:
    7033              :      X = pattern_expr (arg0, arg1, ..., X)
    7034              :    and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
    7035              :    sequence that had been detected and replaced by the pattern-stmt
    7036              :    (STMT_INFO).
    7037              : 
    7038              :    This function also handles reduction of condition expressions, for example:
    7039              :      for (int i = 0; i < N; i++)
    7040              :        if (a[i] < value)
    7041              :          last = a[i];
    7042              :    This is handled by vectorising the loop and creating an additional vector
    7043              :    containing the loop indexes for which "a[i] < value" was true.  In the
    7044              :    function epilogue this is reduced to a single max value and then used to
    7045              :    index into the vector of results.
    7046              : 
    7047              :    In some cases of reduction patterns, the type of the reduction variable X is
    7048              :    different than the type of the other arguments of STMT_INFO.
    7049              :    In such cases, the vectype that is used when transforming STMT_INFO into
    7050              :    a vector stmt is different than the vectype that is used to determine the
    7051              :    vectorization factor, because it consists of a different number of elements
    7052              :    than the actual number of elements that are being operated upon in parallel.
    7053              : 
    7054              :    For example, consider an accumulation of shorts into an int accumulator.
    7055              :    On some targets it's possible to vectorize this pattern operating on 8
    7056              :    shorts at a time (hence, the vectype for purposes of determining the
    7057              :    vectorization factor should be V8HI); on the other hand, the vectype that
    7058              :    is used to create the vector form is actually V4SI (the type of the result).
    7059              : 
    7060              :    Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
    7061              :    indicates what is the actual level of parallelism (V8HI in the example), so
    7062              :    that the right vectorization factor would be derived.  This vectype
    7063              :    corresponds to the type of arguments to the reduction stmt, and should *NOT*
    7064              :    be used to create the vectorized stmt.  The right vectype for the vectorized
    7065              :    stmt is obtained from the type of the result X:
    7066              :       get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
    7067              : 
    7068              :    This means that, contrary to "regular" reductions (or "regular" stmts in
    7069              :    general), the following equation:
    7070              :       STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
    7071              :    does *NOT* necessarily hold for reduction patterns.  */
    7072              : 
    7073              : bool
    7074       414487 : vectorizable_reduction (loop_vec_info loop_vinfo,
    7075              :                         stmt_vec_info stmt_info, slp_tree slp_node,
    7076              :                         slp_instance slp_node_instance,
    7077              :                         stmt_vector_for_cost *cost_vec)
    7078              : {
    7079       414487 :   tree vectype_in = NULL_TREE;
    7080       414487 :   enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
    7081       414487 :   stmt_vec_info cond_stmt_vinfo = NULL;
    7082       414487 :   int i;
    7083       414487 :   int ncopies;
    7084       414487 :   bool single_defuse_cycle = false;
    7085       414487 :   tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE;
    7086       414487 :   tree cond_reduc_val = NULL_TREE;
    7087              : 
    7088              :   /* Make sure it was already recognized as a reduction computation.  */
    7089       414487 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
    7090              :       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def
    7091       414487 :       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
    7092              :     return false;
    7093              : 
    7094              :   /* The reduction meta.  */
    7095        82718 :   vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
    7096              : 
    7097        82718 :   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
    7098              :     {
    7099         1392 :       gcc_assert (is_a <gphi *> (stmt_info->stmt));
    7100              :       /* We eventually need to set a vector type on invariant arguments.  */
    7101              :       unsigned j;
    7102              :       slp_tree child;
    7103         4168 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
    7104         2784 :         if (!vect_maybe_update_slp_op_vectype (child,
    7105              :                                                SLP_TREE_VECTYPE (slp_node)))
    7106              :           {
    7107            0 :             if (dump_enabled_p ())
    7108            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7109              :                                "incompatible vector types for "
    7110              :                                "invariants\n");
    7111              :             return false;
    7112              :           }
    7113         2784 :         else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
    7114         2784 :                  && !useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
    7115              :                                                 SLP_TREE_VECTYPE (child)))
    7116              :           {
    7117              :             /* With bools we can have mask and non-mask precision vectors
    7118              :                or different non-mask precisions.  while pattern recog is
    7119              :                supposed to guarantee consistency here, we do not have
    7120              :                pattern stmts for PHIs (PR123316).
    7121              :                Deal with that here instead of ICEing later.  */
    7122            8 :             if (dump_enabled_p ())
    7123            8 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7124              :                                "incompatible vector type setup from "
    7125              :                                "bool pattern detection\n");
    7126              :             return false;
    7127              :           }
    7128              :       /* Analysis for double-reduction is done on the outer
    7129              :          loop PHI, nested cycles have no further restrictions.  */
    7130         1384 :       SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
    7131         1384 :       return true;
    7132              :     }
    7133              : 
    7134        81326 :   if (!is_a <gphi *> (stmt_info->stmt))
    7135              :     {
    7136         8375 :       gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def);
    7137         8375 :       SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
    7138         8375 :       return true;
    7139              :     }
    7140              : 
    7141        72951 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    7142        72951 :   stmt_vec_info phi_info = stmt_info;
    7143        72951 :   bool double_reduc = false;
    7144        72951 :   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
    7145              :     {
    7146              :       /* We arrive here for both the inner loop LC PHI and the
    7147              :          outer loop PHI.  The latter is what we want to analyze the
    7148              :          reduction with.  The LC PHI is handled by vectorizable_lc_phi.  */
    7149          314 :       if (gimple_bb (stmt_info->stmt) != loop->header)
    7150            0 :         return false;
    7151              : 
    7152              :       /* Set loop and phi_info to the inner loop.  */
    7153          314 :       use_operand_p use_p;
    7154          314 :       gimple *use_stmt;
    7155          314 :       bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
    7156              :                                  &use_p, &use_stmt);
    7157          314 :       gcc_assert (res);
    7158          314 :       phi_info = loop_vinfo->lookup_stmt (use_stmt);
    7159          314 :       loop = loop->inner;
    7160          314 :       double_reduc = true;
    7161              :     }
    7162              : 
    7163        72951 :   const bool reduc_chain = reduc_info->is_reduc_chain;
    7164        72951 :   slp_node_instance->reduc_phis = slp_node;
    7165              :   /* ???  We're leaving slp_node to point to the PHIs, we only
    7166              :      need it to get at the number of vector stmts which wasn't
    7167              :      yet initialized for the instance root.  */
    7168              : 
    7169              :   /* PHIs should not participate in patterns.  */
    7170        72951 :   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
    7171        72951 :   gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
    7172              : 
    7173              :   /* Verify following REDUC_IDX from the latch def leads us back to the PHI
    7174              :      and compute the reduction chain length.  Discover the real
    7175              :      reduction operation stmt on the way (slp_for_stmt_info).  */
    7176        72951 :   unsigned reduc_chain_length = 0;
    7177        72951 :   stmt_info = NULL;
    7178        72951 :   slp_tree slp_for_stmt_info = NULL;
    7179        72951 :   slp_tree vdef_slp = slp_node_instance->root;
    7180       160014 :   while (vdef_slp != slp_node)
    7181              :     {
    7182        88155 :       int reduc_idx = SLP_TREE_REDUC_IDX (vdef_slp);
    7183        88155 :       if (reduc_idx == -1)
    7184              :         {
    7185         1084 :           if (dump_enabled_p ())
    7186            7 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7187              :                              "reduction chain broken by patterns.\n");
    7188         1092 :           return false;
    7189              :         }
    7190        87071 :       stmt_vec_info vdef = SLP_TREE_REPRESENTATIVE (vdef_slp);
    7191        87071 :       if (is_a <gphi *> (vdef->stmt))
    7192              :         {
    7193          628 :           vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
    7194              :           /* Do not count PHIs towards the chain length.  */
    7195          628 :           continue;
    7196              :         }
    7197        86443 :       gimple_match_op op;
    7198        86443 :       if (!gimple_extract_op (vdef->stmt, &op))
    7199              :         {
    7200            0 :           if (dump_enabled_p ())
    7201            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7202              :                              "reduction chain includes unsupported"
    7203              :                              " statement type.\n");
    7204              :           return false;
    7205              :         }
    7206        86443 :       if (CONVERT_EXPR_CODE_P (op.code))
    7207              :         {
    7208         5392 :           if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
    7209              :             {
    7210            8 :               if (dump_enabled_p ())
    7211            8 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7212              :                                  "conversion in the reduction chain.\n");
    7213              :               return false;
    7214              :             }
    7215         5384 :           vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
    7216              :         }
    7217              :       else
    7218              :         {
    7219              :           /* First non-conversion stmt.  */
    7220        81051 :           if (!slp_for_stmt_info)
    7221        71847 :             slp_for_stmt_info = vdef_slp;
    7222              : 
    7223        81051 :           if (lane_reducing_op_p (op.code))
    7224              :             {
    7225              :               /* The last operand of lane-reducing operation is for
    7226              :                  reduction.  */
    7227          740 :               gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
    7228              : 
    7229          740 :               slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
    7230          740 :               tree vectype_op = SLP_TREE_VECTYPE (op_node);
    7231          740 :               tree type_op = TREE_TYPE (op.ops[0]);
    7232          740 :               if (!vectype_op)
    7233              :                 {
    7234           18 :                   vectype_op = get_vectype_for_scalar_type (loop_vinfo,
    7235              :                                                             type_op);
    7236           18 :                   if (!vectype_op
    7237           18 :                       || !vect_maybe_update_slp_op_vectype (op_node,
    7238              :                                                             vectype_op))
    7239              :                     return false;
    7240              :                 }
    7241              : 
    7242              :               /* To accommodate lane-reducing operations of mixed input
    7243              :                  vectypes, choose input vectype with the least lanes for the
    7244              :                  reduction PHI statement, which would result in the most
    7245              :                  ncopies for vectorized reduction results.  */
    7246          740 :               if (!vectype_in
    7247          740 :                   || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
    7248          769 :                        < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
    7249              :                 vectype_in = vectype_op;
    7250              :             }
    7251        80311 :           else if (!vectype_in)
    7252        71136 :             vectype_in = SLP_TREE_VECTYPE (slp_node);
    7253        81051 :           vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
    7254              :         }
    7255        86435 :       reduc_chain_length++;
    7256              :     }
    7257        71859 :   if (!slp_for_stmt_info)
    7258              :     {
    7259           12 :       if (dump_enabled_p ())
    7260           12 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7261              :                          "only noop-conversions in the reduction chain.\n");
    7262              :       return false;
    7263              :     }
    7264        71847 :   stmt_info = SLP_TREE_REPRESENTATIVE (slp_for_stmt_info);
    7265              : 
    7266              :   /* PHIs should not participate in patterns.  */
    7267        71847 :   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
    7268              : 
    7269              :   /* 1. Is vectorizable reduction?  */
    7270              :   /* Not supportable if the reduction variable is used in the loop, unless
    7271              :      it's a reduction chain.  */
    7272        71847 :   if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer
    7273            0 :       && !reduc_chain)
    7274              :     return false;
    7275              : 
    7276              :   /* Reductions that are not used even in an enclosing outer-loop,
    7277              :      are expected to be "live" (used out of the loop).  */
    7278        71847 :   if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
    7279            0 :       && !STMT_VINFO_LIVE_P (stmt_info))
    7280              :     return false;
    7281              : 
    7282              :   /* 2. Has this been recognized as a reduction pattern?
    7283              : 
    7284              :      Check if STMT represents a pattern that has been recognized
    7285              :      in earlier analysis stages.  For stmts that represent a pattern,
    7286              :      the STMT_VINFO_RELATED_STMT field records the last stmt in
    7287              :      the original sequence that constitutes the pattern.  */
    7288              : 
    7289        71847 :   stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
    7290        71847 :   if (orig_stmt_info)
    7291              :     {
    7292         5115 :       gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
    7293         5115 :       gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
    7294              :     }
    7295              : 
    7296              :   /* 3. Check the operands of the operation.  The first operands are defined
    7297              :         inside the loop body. The last operand is the reduction variable,
    7298              :         which is defined by the loop-header-phi.  */
    7299              : 
    7300        71847 :   tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
    7301        71847 :   VECT_REDUC_INFO_VECTYPE (reduc_info) = vectype_out;
    7302              : 
    7303        71847 :   gimple_match_op op;
    7304        71847 :   if (!gimple_extract_op (stmt_info->stmt, &op))
    7305            0 :     gcc_unreachable ();
    7306        71847 :   bool lane_reducing = lane_reducing_op_p (op.code);
    7307              : 
    7308        71847 :   if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
    7309        19097 :       && !SCALAR_FLOAT_TYPE_P (op.type))
    7310              :     return false;
    7311              : 
    7312              :   /* Do not try to vectorize bit-precision reductions.  */
    7313        71847 :   if (!type_has_mode_precision_p (op.type)
    7314         1783 :       && op.code != BIT_AND_EXPR
    7315         1653 :       && op.code != BIT_IOR_EXPR
    7316        72339 :       && op.code != BIT_XOR_EXPR)
    7317              :     return false;
    7318              : 
    7319              :   /* Lane-reducing ops also never can be used in a SLP reduction group
    7320              :      since we'll mix lanes belonging to different reductions.  But it's
    7321              :      OK to use them in a reduction chain or when the reduction group
    7322              :      has just one element.  */
    7323        71537 :   if (lane_reducing
    7324        71537 :       && !reduc_chain
    7325          668 :       && SLP_TREE_LANES (slp_node) > 1)
    7326              :     {
    7327            0 :       if (dump_enabled_p ())
    7328            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7329              :                          "lane-reducing reduction in reduction group.\n");
    7330              :       return false;
    7331              :     }
    7332              : 
    7333              :   /* We'll verify the reduction operation only later - avoid
    7334              :      all operations that mismatch on the number of SLP children.  */
    7335       143074 :   if (op.num_ops != SLP_TREE_CHILDREN (slp_for_stmt_info).length ())
    7336              :     {
    7337            0 :       if (dump_enabled_p ())
    7338            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7339              :                          "unsupported reduction operation.\n");
    7340              :       return false;
    7341              :     }
    7342              : 
    7343              :   /* All uses but the last are expected to be defined in the loop.
    7344              :      The last use is the reduction variable.  In case of nested cycle this
    7345              :      assumption is not true: we use reduc_index to record the index of the
    7346              :      reduction variable.  */
    7347        71537 :   slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
    7348        71537 :   tree *vectype_op = XALLOCAVEC (tree, op.num_ops);
    7349        71537 :   gcc_assert (op.code != COND_EXPR || !COMPARISON_CLASS_P (op.ops[0]));
    7350       229712 :   for (i = 0; i < (int) op.num_ops; i++)
    7351              :     {
    7352              :       /* The condition of COND_EXPR is checked in vectorizable_condition().  */
    7353       158175 :       if (i == 0 && op.code == COND_EXPR)
    7354        79169 :         continue;
    7355              : 
    7356       157302 :       stmt_vec_info def_stmt_info;
    7357       157302 :       enum vect_def_type dt;
    7358       157302 :       if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
    7359              :                                i, &op.ops[i], &slp_op[i], &dt,
    7360       157302 :                                &vectype_op[i], &def_stmt_info))
    7361              :         {
    7362            0 :           if (dump_enabled_p ())
    7363            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7364              :                              "use not simple.\n");
    7365            0 :           return false;
    7366              :         }
    7367              : 
    7368              :       /* Skip reduction operands, and for an IFN_COND_OP we might hit the
    7369              :          reduction operand twice (once as definition, once as else).  */
    7370       157302 :       if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]
    7371       314604 :           == SLP_TREE_CHILDREN
    7372       157302 :                (slp_for_stmt_info)[SLP_TREE_REDUC_IDX (slp_for_stmt_info)])
    7373        78296 :         continue;
    7374              : 
    7375              :       /* There should be only one cycle def in the stmt, the one
    7376              :          leading to reduc_def.  */
    7377        79006 :       if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]->cycle_info.id != -1)
    7378              :         return false;
    7379              : 
    7380        79006 :       if (!vectype_op[i])
    7381         7458 :         vectype_op[i]
    7382         7458 :           = get_vectype_for_scalar_type (loop_vinfo,
    7383         7458 :                                          TREE_TYPE (op.ops[i]), slp_op[i]);
    7384              : 
    7385              :       /* Record how the non-reduction-def value of COND_EXPR is defined.
    7386              :          ???  For a chain of multiple CONDs we'd have to match them up all.  */
    7387        79006 :       if (op.code == COND_EXPR && reduc_chain_length == 1)
    7388              :         {
    7389          850 :           if (dt == vect_constant_def)
    7390              :             {
    7391          124 :               cond_reduc_dt = dt;
    7392          124 :               cond_reduc_val = op.ops[i];
    7393              :             }
    7394          726 :           else if (dt == vect_induction_def
    7395          414 :                    && def_stmt_info
    7396         1140 :                    && is_nonwrapping_integer_induction (def_stmt_info, loop))
    7397              :             {
    7398          109 :               cond_reduc_dt = dt;
    7399          109 :               cond_stmt_vinfo = def_stmt_info;
    7400              :             }
    7401              :         }
    7402              :     }
    7403              : 
    7404        71537 :   enum vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
    7405              :   /* If we have a condition reduction, see if we can simplify it further.  */
    7406        71537 :   if (reduction_type == COND_REDUCTION)
    7407              :     {
    7408          861 :       if (SLP_TREE_LANES (slp_node) != 1)
    7409              :         return false;
    7410              : 
    7411              :       /* When the condition uses the reduction value in the condition, fail.  */
    7412          837 :       if (SLP_TREE_REDUC_IDX (slp_node) == 0)
    7413              :         {
    7414            0 :           if (dump_enabled_p ())
    7415            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7416              :                              "condition depends on previous iteration\n");
    7417              :           return false;
    7418              :         }
    7419              : 
    7420          837 :       if (reduc_chain_length == 1
    7421          837 :           && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in,
    7422              :                                               OPTIMIZE_FOR_SPEED)
    7423          814 :               || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
    7424              :                                                  vectype_in,
    7425              :                                                  OPTIMIZE_FOR_SPEED)))
    7426              :         {
    7427            0 :           if (dump_enabled_p ())
    7428            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7429              :                              "optimizing condition reduction with"
    7430              :                              " FOLD_EXTRACT_LAST.\n");
    7431            0 :           VECT_REDUC_INFO_TYPE (reduc_info) = EXTRACT_LAST_REDUCTION;
    7432              :         }
    7433          837 :       else if (cond_reduc_dt == vect_induction_def)
    7434              :         {
    7435          109 :           tree base
    7436              :             = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo);
    7437          109 :           tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo);
    7438              : 
    7439          109 :           gcc_assert (TREE_CODE (base) == INTEGER_CST
    7440              :                       && TREE_CODE (step) == INTEGER_CST);
    7441          109 :           cond_reduc_val = NULL_TREE;
    7442          109 :           enum tree_code cond_reduc_op_code = ERROR_MARK;
    7443          109 :           tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
    7444          109 :           if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
    7445              :             ;
    7446              :           /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
    7447              :              above base; punt if base is the minimum value of the type for
    7448              :              MAX_EXPR or maximum value of the type for MIN_EXPR for now.  */
    7449           97 :           else if (tree_int_cst_sgn (step) == -1)
    7450              :             {
    7451           18 :               cond_reduc_op_code = MIN_EXPR;
    7452           18 :               if (tree_int_cst_sgn (base) == -1)
    7453            0 :                 cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
    7454           18 :               else if (tree_int_cst_lt (base,
    7455           18 :                                         TYPE_MAX_VALUE (TREE_TYPE (base))))
    7456           18 :                 cond_reduc_val
    7457           18 :                   = int_const_binop (PLUS_EXPR, base, integer_one_node);
    7458              :             }
    7459              :           else
    7460              :             {
    7461           79 :               cond_reduc_op_code = MAX_EXPR;
    7462           79 :               if (tree_int_cst_sgn (base) == 1)
    7463            0 :                 cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
    7464           79 :               else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base)),
    7465              :                                         base))
    7466           79 :                 cond_reduc_val
    7467           79 :                   = int_const_binop (MINUS_EXPR, base, integer_one_node);
    7468              :             }
    7469           97 :           if (cond_reduc_val)
    7470              :             {
    7471           97 :               if (dump_enabled_p ())
    7472           61 :                 dump_printf_loc (MSG_NOTE, vect_location,
    7473              :                                  "condition expression based on "
    7474              :                                  "integer induction.\n");
    7475           97 :               VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_op_code;
    7476           97 :               VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info)
    7477           97 :                 = cond_reduc_val;
    7478           97 :               VECT_REDUC_INFO_TYPE (reduc_info) = INTEGER_INDUC_COND_REDUCTION;
    7479              :             }
    7480              :         }
    7481          728 :       else if (cond_reduc_dt == vect_constant_def)
    7482              :         {
    7483          114 :           enum vect_def_type cond_initial_dt;
    7484          114 :           tree cond_initial_val = vect_phi_initial_value (reduc_def_phi);
    7485          114 :           vect_is_simple_use (cond_initial_val, loop_vinfo, &cond_initial_dt);
    7486          114 :           if (cond_initial_dt == vect_constant_def
    7487          145 :               && types_compatible_p (TREE_TYPE (cond_initial_val),
    7488           31 :                                      TREE_TYPE (cond_reduc_val)))
    7489              :             {
    7490           31 :               tree lt, gt;
    7491           31 :               enum tree_code cond_reduc_code = ERROR_MARK;
    7492           31 :               if ((lt = fold_binary (LT_EXPR, boolean_type_node,
    7493              :                                      cond_initial_val, cond_reduc_val))
    7494           31 :                   && integer_onep (lt))
    7495              :                 cond_reduc_code = MAX_EXPR;
    7496           13 :               else if ((gt = fold_binary (GT_EXPR, boolean_type_node,
    7497              :                                           cond_initial_val, cond_reduc_val))
    7498           13 :                        && integer_onep (gt))
    7499              :                 cond_reduc_code = MIN_EXPR;
    7500              :               if (cond_reduc_code != ERROR_MARK)
    7501              :                 {
    7502           25 :                   if (dump_enabled_p ())
    7503           16 :                     dump_printf_loc (MSG_NOTE, vect_location,
    7504              :                                      "condition expression based on "
    7505              :                                      "compile time constant.\n");
    7506              :                   /* Record reduction code at analysis stage.  */
    7507           25 :                   VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_code;
    7508           25 :                   VECT_REDUC_INFO_TYPE (reduc_info) = CONST_COND_REDUCTION;
    7509              :                 }
    7510              :             }
    7511              :         }
    7512              :     }
    7513              : 
    7514        71513 :   if (STMT_VINFO_LIVE_P (phi_info))
    7515              :     return false;
    7516              : 
    7517        71513 :   ncopies = vect_get_num_copies (loop_vinfo, slp_node);
    7518              : 
    7519        71513 :   gcc_assert (ncopies >= 1);
    7520              : 
    7521        71513 :   poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
    7522              : 
    7523              :   /* 4.2. Check support for the epilog operation.
    7524              : 
    7525              :           If STMT represents a reduction pattern, then the type of the
    7526              :           reduction variable may be different than the type of the rest
    7527              :           of the arguments.  For example, consider the case of accumulation
    7528              :           of shorts into an int accumulator; The original code:
    7529              :                         S1: int_a = (int) short_a;
    7530              :           orig_stmt->   S2: int_acc = plus <int_a ,int_acc>;
    7531              : 
    7532              :           was replaced with:
    7533              :                         STMT: int_acc = widen_sum <short_a, int_acc>
    7534              : 
    7535              :           This means that:
    7536              :           1. The tree-code that is used to create the vector operation in the
    7537              :              epilog code (that reduces the partial results) is not the
    7538              :              tree-code of STMT, but is rather the tree-code of the original
    7539              :              stmt from the pattern that STMT is replacing.  I.e, in the example
    7540              :              above we want to use 'widen_sum' in the loop, but 'plus' in the
    7541              :              epilog.
    7542              :           2. The type (mode) we use to check available target support
    7543              :              for the vector operation to be created in the *epilog*, is
    7544              :              determined by the type of the reduction variable (in the example
    7545              :              above we'd check this: optab_handler (plus_optab, vect_int_mode])).
    7546              :              However the type (mode) we use to check available target support
    7547              :              for the vector operation to be created *inside the loop*, is
    7548              :              determined by the type of the other arguments to STMT (in the
    7549              :              example we'd check this: optab_handler (widen_sum_optab,
    7550              :              vect_short_mode)).
    7551              : 
    7552              :           This is contrary to "regular" reductions, in which the types of all
    7553              :           the arguments are the same as the type of the reduction variable.
    7554              :           For "regular" reductions we can therefore use the same vector type
    7555              :           (and also the same tree-code) when generating the epilog code and
    7556              :           when generating the code inside the loop.  */
    7557              : 
    7558        71513 :   code_helper orig_code = VECT_REDUC_INFO_CODE (reduc_info);
    7559              : 
    7560              :   /* If conversion might have created a conditional operation like
    7561              :      IFN_COND_ADD already.  Use the internal code for the following checks.  */
    7562        71513 :   if (orig_code.is_internal_fn ())
    7563              :     {
    7564         6835 :       tree_code new_code = conditional_internal_fn_code (internal_fn (orig_code));
    7565         6835 :       orig_code = new_code != ERROR_MARK ? new_code : orig_code;
    7566              :     }
    7567              : 
    7568        71513 :   VECT_REDUC_INFO_CODE (reduc_info) = orig_code;
    7569              : 
    7570        71513 :   reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
    7571        71513 :   if (reduction_type == TREE_CODE_REDUCTION)
    7572              :     {
    7573              :       /* Check whether it's ok to change the order of the computation.
    7574              :          Generally, when vectorizing a reduction we change the order of the
    7575              :          computation.  This may change the behavior of the program in some
    7576              :          cases, so we need to check that this is ok.  One exception is when
    7577              :          vectorizing an outer-loop: the inner-loop is executed sequentially,
    7578              :          and therefore vectorizing reductions in the inner-loop during
    7579              :          outer-loop vectorization is safe.  Likewise when we are vectorizing
    7580              :          a series of reductions using SLP and the VF is one the reductions
    7581              :          are performed in scalar order.  */
    7582        70676 :       if (!reduc_chain
    7583        70676 :           && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
    7584              :         ;
    7585        70504 :       else if (needs_fold_left_reduction_p (op.type, orig_code))
    7586              :         {
    7587              :           /* When vectorizing a reduction chain w/o SLP the reduction PHI
    7588              :              is not directly used in stmt.  */
    7589         5654 :           if (reduc_chain_length != 1)
    7590              :             {
    7591          105 :               if (dump_enabled_p ())
    7592           28 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7593              :                                  "in-order reduction chain without SLP.\n");
    7594              :               return false;
    7595              :             }
    7596              :           /* Code generation doesn't support function calls other
    7597              :              than .COND_*.  */
    7598         5549 :           if (!op.code.is_tree_code ()
    7599         5737 :               && !(op.code.is_internal_fn ()
    7600           94 :                    && conditional_internal_fn_code (internal_fn (op.code))
    7601              :                         != ERROR_MARK))
    7602              :             {
    7603           18 :               if (dump_enabled_p ())
    7604           16 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7605              :                                  "in-order reduction chain operation not "
    7606              :                                  "supported.\n");
    7607              :               return false;
    7608              :             }
    7609         5531 :           VECT_REDUC_INFO_TYPE (reduc_info)
    7610         5531 :             = reduction_type = FOLD_LEFT_REDUCTION;
    7611              :         }
    7612        64850 :       else if (!commutative_binary_op_p (orig_code, op.type)
    7613        64850 :                || !associative_binary_op_p (orig_code, op.type))
    7614              :         {
    7615          180 :           if (dump_enabled_p ())
    7616           28 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7617              :                             "reduction: not commutative/associative\n");
    7618              :           return false;
    7619              :         }
    7620              :     }
    7621              : 
    7622         5531 :   if ((reduction_type == COND_REDUCTION
    7623              :        || reduction_type == INTEGER_INDUC_COND_REDUCTION
    7624              :        || reduction_type == CONST_COND_REDUCTION
    7625        65679 :        || reduction_type == EXTRACT_LAST_REDUCTION)
    7626          837 :       && ncopies > 1)
    7627              :     {
    7628          276 :       if (dump_enabled_p ())
    7629           60 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7630              :                          "multiple types in condition reduction.\n");
    7631              :       return false;
    7632              :     }
    7633              : 
    7634              :   /* See if we can convert a mask vector to a corresponding bool data vector
    7635              :      to perform the epilogue reduction.  */
    7636        70934 :   tree alt_vectype_out = NULL_TREE;
    7637        70934 :   if (VECTOR_BOOLEAN_TYPE_P (vectype_out))
    7638              :     {
    7639         1179 :       alt_vectype_out
    7640         2358 :         = get_related_vectype_for_scalar_type (loop_vinfo->vector_mode,
    7641         1179 :                                                TREE_TYPE (vectype_out),
    7642              :                                                TYPE_VECTOR_SUBPARTS
    7643              :                                                  (vectype_out));
    7644         1179 :       if (!alt_vectype_out
    7645         1179 :           || maybe_ne (TYPE_VECTOR_SUBPARTS (alt_vectype_out),
    7646         2358 :                        TYPE_VECTOR_SUBPARTS (vectype_out))
    7647         2358 :           || !expand_vec_cond_expr_p (alt_vectype_out, vectype_out))
    7648              :         alt_vectype_out = NULL_TREE;
    7649              :     }
    7650              : 
    7651        70934 :   internal_fn reduc_fn = IFN_LAST;
    7652        70934 :   if (reduction_type == TREE_CODE_REDUCTION
    7653        70934 :       || reduction_type == FOLD_LEFT_REDUCTION
    7654              :       || reduction_type == INTEGER_INDUC_COND_REDUCTION
    7655          561 :       || reduction_type == CONST_COND_REDUCTION)
    7656              :     {
    7657        64956 :       if (reduction_type == FOLD_LEFT_REDUCTION
    7658        75225 :           ? fold_left_reduction_fn (orig_code, &reduc_fn)
    7659        64956 :           : reduction_fn_for_scalar_code (orig_code, &reduc_fn))
    7660              :         {
    7661        69808 :           internal_fn sbool_fn = IFN_LAST;
    7662        69808 :           if (reduc_fn == IFN_LAST)
    7663              :             ;
    7664        67714 :           else if ((!VECTOR_BOOLEAN_TYPE_P (vectype_out)
    7665         1179 :                     || (GET_MODE_CLASS (TYPE_MODE (vectype_out))
    7666              :                         == MODE_VECTOR_BOOL))
    7667       134249 :                    && direct_internal_fn_supported_p (reduc_fn, vectype_out,
    7668              :                                                       OPTIMIZE_FOR_SPEED))
    7669              :             ;
    7670        18636 :           else if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
    7671         1179 :                    && sbool_reduction_fn_for_fn (reduc_fn, &sbool_fn)
    7672        19815 :                    && direct_internal_fn_supported_p (sbool_fn, vectype_out,
    7673              :                                                       OPTIMIZE_FOR_SPEED))
    7674          131 :             reduc_fn = sbool_fn;
    7675        18505 :           else if (reduction_type != FOLD_LEFT_REDUCTION
    7676        18505 :                    && alt_vectype_out
    7677        18505 :                    && direct_internal_fn_supported_p (reduc_fn, alt_vectype_out,
    7678              :                                                       OPTIMIZE_FOR_SPEED))
    7679          805 :             VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
    7680              :           else
    7681              :             {
    7682        17700 :               if (dump_enabled_p ())
    7683          966 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7684              :                                  "reduc op not supported by target.\n");
    7685              : 
    7686        17700 :               reduc_fn = IFN_LAST;
    7687              :             }
    7688              :         }
    7689              :       else
    7690              :         {
    7691          679 :           if (dump_enabled_p ())
    7692           48 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7693              :                              "no reduc code for scalar code.\n");
    7694              : 
    7695              :           return false;
    7696              :         }
    7697        69808 :       if (reduc_fn == IFN_LAST
    7698        69808 :           && VECTOR_BOOLEAN_TYPE_P (vectype_out))
    7699              :         {
    7700          243 :           if (!alt_vectype_out)
    7701              :             {
    7702           12 :               if (dump_enabled_p ())
    7703            8 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7704              :                                  "cannot turn mask into bool data vector for "
    7705              :                                  "reduction epilogue.\n");
    7706              :               return false;
    7707              :             }
    7708          231 :           VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
    7709              :         }
    7710              :     }
    7711          447 :   else if (reduction_type == COND_REDUCTION)
    7712              :     {
    7713          447 :       int scalar_precision
    7714          447 :         = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
    7715          447 :       cr_index_scalar_type = make_unsigned_type (scalar_precision);
    7716          447 :       cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
    7717              :                                                 vectype_out);
    7718              : 
    7719          447 :       if (direct_internal_fn_supported_p (IFN_REDUC_MAX, cr_index_vector_type,
    7720              :                                           OPTIMIZE_FOR_SPEED))
    7721           22 :         reduc_fn = IFN_REDUC_MAX;
    7722              :     }
    7723        70243 :   VECT_REDUC_INFO_FN (reduc_info) = reduc_fn;
    7724              : 
    7725        70243 :   if (reduction_type != EXTRACT_LAST_REDUCTION
    7726              :       && reduc_fn == IFN_LAST
    7727              :       && !nunits_out.is_constant ())
    7728              :     {
    7729              :       if (dump_enabled_p ())
    7730              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7731              :                          "missing target support for reduction on"
    7732              :                          " variable-length vectors.\n");
    7733              :       return false;
    7734              :     }
    7735              : 
    7736              :   /* For SLP reductions, see if there is a neutral value we can use.  */
    7737        70243 :   tree neutral_op = NULL_TREE;
    7738        70243 :   tree initial_value = NULL_TREE;
    7739        70243 :   if (reduc_chain)
    7740          522 :     initial_value = vect_phi_initial_value (reduc_def_phi);
    7741        70243 :   neutral_op = neutral_op_for_reduction (TREE_TYPE
    7742              :                                            (gimple_phi_result (reduc_def_phi)),
    7743              :                                          orig_code, initial_value);
    7744        70243 :   VECT_REDUC_INFO_NEUTRAL_OP (reduc_info) = neutral_op;
    7745              : 
    7746        70243 :   if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
    7747              :     {
    7748              :       /* We can't support in-order reductions of code such as this:
    7749              : 
    7750              :            for (int i = 0; i < n1; ++i)
    7751              :              for (int j = 0; j < n2; ++j)
    7752              :                l += a[j];
    7753              : 
    7754              :          since GCC effectively transforms the loop when vectorizing:
    7755              : 
    7756              :            for (int i = 0; i < n1 / VF; ++i)
    7757              :              for (int j = 0; j < n2; ++j)
    7758              :                for (int k = 0; k < VF; ++k)
    7759              :                  l += a[j];
    7760              : 
    7761              :          which is a reassociation of the original operation.  */
    7762           66 :       if (dump_enabled_p ())
    7763           20 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7764              :                          "in-order double reduction not supported.\n");
    7765              : 
    7766              :       return false;
    7767              :     }
    7768              : 
    7769        70177 :   if (reduction_type == FOLD_LEFT_REDUCTION
    7770         4786 :       && SLP_TREE_LANES (slp_node) > 1
    7771          194 :       && !reduc_chain)
    7772              :     {
    7773              :       /* We cannot use in-order reductions in this case because there is
    7774              :          an implicit reassociation of the operations involved.  */
    7775           72 :       if (dump_enabled_p ())
    7776            6 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7777              :                          "in-order unchained SLP reductions not supported.\n");
    7778              :       return false;
    7779              :     }
    7780              : 
    7781              :   /* For double reductions, and for SLP reductions with a neutral value,
    7782              :      we construct a variable-length initial vector by loading a vector
    7783              :      full of the neutral value and then shift-and-inserting the start
    7784              :      values into the low-numbered elements.  This is however not needed
    7785              :      when neutral and initial value are equal or we can handle the
    7786              :      initial value via adjustment in the epilogue.  */
    7787        70105 :   if ((double_reduc || neutral_op)
    7788              :       && !nunits_out.is_constant ()
    7789              :       && reduction_type != INTEGER_INDUC_COND_REDUCTION
    7790              :       && !((SLP_TREE_LANES (slp_node) == 1 || reduc_chain)
    7791              :            && neutral_op
    7792              :            && (!double_reduc
    7793              :                || operand_equal_p (neutral_op,
    7794              :                                    vect_phi_initial_value (reduc_def_phi))))
    7795              :       && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT,
    7796              :                                           vectype_out, OPTIMIZE_FOR_BOTH))
    7797              :     {
    7798              :       if (dump_enabled_p ())
    7799              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7800              :                          "reduction on variable-length vectors requires"
    7801              :                          " target support for a vector-shift-and-insert"
    7802              :                          " operation.\n");
    7803              :       return false;
    7804              :     }
    7805              : 
    7806              :   /* Check extra constraints for variable-length unchained SLP reductions.  */
    7807        70105 :   if (!reduc_chain
    7808              :       && !nunits_out.is_constant ())
    7809              :     {
    7810              :       /* We checked above that we could build the initial vector when
    7811              :          there's a neutral element value.  Check here for the case in
    7812              :          which each SLP statement has its own initial value and in which
    7813              :          that value needs to be repeated for every instance of the
    7814              :          statement within the initial vector.  */
    7815              :       unsigned int group_size = SLP_TREE_LANES (slp_node);
    7816              :       if (!neutral_op
    7817              :           && !can_duplicate_and_interleave_p (loop_vinfo, group_size,
    7818              :                                               TREE_TYPE (vectype_out)))
    7819              :         {
    7820              :           if (dump_enabled_p ())
    7821              :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7822              :                              "unsupported form of SLP reduction for"
    7823              :                              " variable-length vectors: cannot build"
    7824              :                              " initial vector.\n");
    7825              :           return false;
    7826              :         }
    7827              :       /* The epilogue code relies on the number of elements being a multiple
    7828              :          of the group size.  The duplicate-and-interleave approach to setting
    7829              :          up the initial vector does too.  */
    7830              :       if (!multiple_p (nunits_out, group_size))
    7831              :         {
    7832              :           if (dump_enabled_p ())
    7833              :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7834              :                              "unsupported form of SLP reduction for"
    7835              :                              " variable-length vectors: the vector size"
    7836              :                              " is not a multiple of the number of results.\n");
    7837              :           return false;
    7838              :         }
    7839              :     }
    7840              : 
    7841        70105 :   if (reduction_type == COND_REDUCTION)
    7842              :     {
    7843          447 :       widest_int ni;
    7844              : 
    7845          447 :       if (! max_loop_iterations (loop, &ni))
    7846              :         {
    7847            8 :           if (dump_enabled_p ())
    7848            0 :             dump_printf_loc (MSG_NOTE, vect_location,
    7849              :                              "loop count not known, cannot create cond "
    7850              :                              "reduction.\n");
    7851              :           return false;
    7852              :         }
    7853              :       /* Convert backedges to iterations.  */
    7854          439 :       ni += 1;
    7855              : 
    7856              :       /* The additional index will be the same type as the condition.  Check
    7857              :          that the loop can fit into this less one (because we'll use up the
    7858              :          zero slot for when there are no matches).  */
    7859          439 :       tree max_index = TYPE_MAX_VALUE (cr_index_scalar_type);
    7860          439 :       if (wi::geu_p (ni, wi::to_widest (max_index)))
    7861              :         {
    7862           90 :           if (dump_enabled_p ())
    7863           54 :             dump_printf_loc (MSG_NOTE, vect_location,
    7864              :                              "loop size is greater than data size.\n");
    7865              :           return false;
    7866              :         }
    7867          447 :     }
    7868              : 
    7869              :   /* In case the vectorization factor (VF) is bigger than the number
    7870              :      of elements that we can fit in a vectype (nunits), we have to generate
    7871              :      more than one vector stmt - i.e - we need to "unroll" the
    7872              :      vector stmt by a factor VF/nunits.  For more details see documentation
    7873              :      in vectorizable_operation.  */
    7874              : 
    7875              :   /* If the reduction is used in an outer loop we need to generate
    7876              :      VF intermediate results, like so (e.g. for ncopies=2):
    7877              :         r0 = phi (init, r0)
    7878              :         r1 = phi (init, r1)
    7879              :         r0 = x0 + r0;
    7880              :         r1 = x1 + r1;
    7881              :     (i.e. we generate VF results in 2 registers).
    7882              :     In this case we have a separate def-use cycle for each copy, and therefore
    7883              :     for each copy we get the vector def for the reduction variable from the
    7884              :     respective phi node created for this copy.
    7885              : 
    7886              :     Otherwise (the reduction is unused in the loop nest), we can combine
    7887              :     together intermediate results, like so (e.g. for ncopies=2):
    7888              :         r = phi (init, r)
    7889              :         r = x0 + r;
    7890              :         r = x1 + r;
    7891              :    (i.e. we generate VF/2 results in a single register).
    7892              :    In this case for each copy we get the vector def for the reduction variable
    7893              :    from the vectorized reduction operation generated in the previous iteration.
    7894              : 
    7895              :    This only works when we see both the reduction PHI and its only consumer
    7896              :    in vectorizable_reduction and there are no intermediate stmts
    7897              :    participating.  When unrolling we want each unrolled iteration to have its
    7898              :    own reduction accumulator since one of the main goals of unrolling a
    7899              :    reduction is to reduce the aggregate loop-carried latency.  */
    7900        70007 :   if (ncopies > 1
    7901        70007 :       && !reduc_chain
    7902         8156 :       && SLP_TREE_LANES (slp_node) == 1
    7903         7983 :       && (STMT_VINFO_RELEVANT (stmt_info) <= vect_used_only_live)
    7904         7960 :       && reduc_chain_length == 1
    7905         7558 :       && loop_vinfo->suggested_unroll_factor == 1)
    7906        70007 :     single_defuse_cycle = true;
    7907              : 
    7908        70007 :   if (single_defuse_cycle && !lane_reducing)
    7909              :     {
    7910         6601 :       gcc_assert (op.code != COND_EXPR);
    7911              : 
    7912              :       /* 4. check support for the operation in the loop
    7913              : 
    7914              :          This isn't necessary for the lane reduction codes, since they
    7915              :          can only be produced by pattern matching, and it's up to the
    7916              :          pattern matcher to test for support.  The main reason for
    7917              :          specifically skipping this step is to avoid rechecking whether
    7918              :          mixed-sign dot-products can be implemented using signed
    7919              :          dot-products.  */
    7920         6601 :       machine_mode vec_mode = TYPE_MODE (vectype_in);
    7921         6601 :       if (!directly_supported_p (op.code, vectype_in, optab_vector))
    7922              :         {
    7923         2076 :           if (dump_enabled_p ())
    7924           36 :             dump_printf (MSG_NOTE, "op not supported by target.\n");
    7925         4152 :           if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
    7926         2076 :               || !vect_can_vectorize_without_simd_p (op.code))
    7927              :             single_defuse_cycle = false;
    7928              :           else
    7929           10 :             if (dump_enabled_p ())
    7930            0 :               dump_printf (MSG_NOTE, "proceeding using word mode.\n");
    7931              :         }
    7932              : 
    7933         6601 :       if (vect_emulated_vector_p (vectype_in)
    7934         6601 :           && !vect_can_vectorize_without_simd_p (op.code))
    7935              :         {
    7936            0 :           if (dump_enabled_p ())
    7937            0 :             dump_printf (MSG_NOTE, "using word mode not possible.\n");
    7938              :           return false;
    7939              :         }
    7940              :     }
    7941        70007 :   if (dump_enabled_p () && single_defuse_cycle)
    7942          731 :     dump_printf_loc (MSG_NOTE, vect_location,
    7943              :                      "using single def-use cycle for reduction by reducing "
    7944              :                      "multiple vectors to one in the loop body\n");
    7945        70007 :   VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info) = single_defuse_cycle;
    7946              : 
    7947              :   /* For lane-reducing operation, the below processing related to single
    7948              :      defuse-cycle will be done in its own vectorizable function.  One more
    7949              :      thing to note is that the operation must not be involved in fold-left
    7950              :      reduction.  */
    7951        70007 :   single_defuse_cycle &= !lane_reducing;
    7952              : 
    7953        70007 :   if (single_defuse_cycle || reduction_type == FOLD_LEFT_REDUCTION)
    7954        29842 :     for (i = 0; i < (int) op.num_ops; i++)
    7955        20688 :       if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_op[i]))
    7956              :         {
    7957            0 :           if (dump_enabled_p ())
    7958            0 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    7959              :                              "incompatible vector types for invariants\n");
    7960              :           return false;
    7961              :         }
    7962              : 
    7963        70007 :   vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
    7964              :                              reduction_type, ncopies, cost_vec);
    7965              :   /* Cost the reduction op inside the loop if transformed via
    7966              :      vect_transform_reduction for non-lane-reducing operation.  Otherwise
    7967              :      this is costed by the separate vectorizable_* routines.  */
    7968        70007 :   if (single_defuse_cycle)
    7969         4535 :     record_stmt_cost (cost_vec, ncopies, vector_stmt,
    7970              :                       slp_for_stmt_info, 0, vect_body);
    7971              : 
    7972        70007 :   if (dump_enabled_p ()
    7973        70007 :       && reduction_type == FOLD_LEFT_REDUCTION)
    7974          264 :     dump_printf_loc (MSG_NOTE, vect_location,
    7975              :                      "using an in-order (fold-left) reduction.\n");
    7976        70007 :   SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
    7977              : 
    7978              :   /* All but single defuse-cycle optimized and fold-left reductions go
    7979              :      through their own vectorizable_* routines.  */
    7980        70007 :   stmt_vec_info tem
    7981        70007 :     = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (slp_node_instance));
    7982        70007 :   if (!single_defuse_cycle && reduction_type != FOLD_LEFT_REDUCTION)
    7983        60853 :     STMT_VINFO_DEF_TYPE (tem) = vect_internal_def;
    7984              :   else
    7985              :     {
    7986         9154 :       STMT_VINFO_DEF_TYPE (tem) = vect_reduction_def;
    7987         9154 :       if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
    7988         9154 :           && vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
    7989              :                                                          op.code, op.type,
    7990              :                                                          vectype_in))
    7991              :         {
    7992         2098 :           internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
    7993         2098 :           internal_fn mask_reduc_fn
    7994         2098 :             = get_masked_reduction_fn (reduc_fn, vectype_in);
    7995         2098 :           vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    7996         2098 :           vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
    7997         2098 :           if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
    7998            0 :             vect_record_loop_len (loop_vinfo, lens, ncopies, vectype_in, 1);
    7999              :           else
    8000         2098 :             vect_record_loop_mask (loop_vinfo, masks, ncopies,
    8001              :                                    vectype_in, NULL);
    8002              :         }
    8003              :     }
    8004              :   return true;
    8005              : }
    8006              : 
    8007              : /* STMT_INFO is a dot-product reduction whose multiplication operands
    8008              :    have different signs.  Emit a sequence to emulate the operation
    8009              :    using a series of signed DOT_PROD_EXPRs and return the last
    8010              :    statement generated.  VEC_DEST is the result of the vector operation
    8011              :    and VOP lists its inputs.  */
    8012              : 
    8013              : static gassign *
    8014            4 : vect_emulate_mixed_dot_prod (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
    8015              :                              gimple_stmt_iterator *gsi, tree vec_dest,
    8016              :                              tree vop[3])
    8017              : {
    8018            4 :   tree wide_vectype = signed_type_for (TREE_TYPE (vec_dest));
    8019            4 :   tree narrow_vectype = signed_type_for (TREE_TYPE (vop[0]));
    8020            4 :   tree narrow_elttype = TREE_TYPE (narrow_vectype);
    8021            4 :   gimple *new_stmt;
    8022              : 
    8023              :   /* Make VOP[0] the unsigned operand VOP[1] the signed operand.  */
    8024            4 :   if (!TYPE_UNSIGNED (TREE_TYPE (vop[0])))
    8025            0 :     std::swap (vop[0], vop[1]);
    8026              : 
    8027              :   /* Convert all inputs to signed types.  */
    8028           12 :   for (int i = 1; i < 3; ++i)
    8029            8 :     if (TYPE_UNSIGNED (TREE_TYPE (vop[i])))
    8030              :       {
    8031            0 :         tree tmp = make_ssa_name (signed_type_for (TREE_TYPE (vop[i])));
    8032            0 :         new_stmt = gimple_build_assign (tmp, NOP_EXPR, vop[i]);
    8033            0 :         vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8034            0 :         vop[i] = tmp;
    8035              :       }
    8036              : 
    8037              :   /* In the comments below we assume 8-bit inputs for simplicity,
    8038              :      but the approach works for any full integer type.  */
    8039              : 
    8040              :   /* Create a vector of -128.  */
    8041            4 :   tree min_narrow_elttype = TYPE_MIN_VALUE (narrow_elttype);
    8042            4 :   tree min_narrow = build_vector_from_val (TREE_TYPE (vop[0]),
    8043            4 :                                            fold_convert
    8044              :                                              (TREE_TYPE (TREE_TYPE (vop[0])),
    8045              :                                               min_narrow_elttype));
    8046              : 
    8047              :   /* Create a vector of 64.  */
    8048            4 :   auto half_wi = wi::lrshift (wi::to_wide (min_narrow_elttype), 1);
    8049            4 :   tree half_narrow = wide_int_to_tree (narrow_elttype, half_wi);
    8050            4 :   half_narrow = build_vector_from_val (narrow_vectype, half_narrow);
    8051              : 
    8052              :   /* Emit: SUB_RES = VOP[0] - 128 in an unsigned type.  */
    8053            4 :   tree sub_res = make_ssa_name (TREE_TYPE (vop[0]));
    8054            4 :   new_stmt = gimple_build_assign (sub_res, PLUS_EXPR, vop[0], min_narrow);
    8055            4 :   vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8056              : 
    8057            4 :   vop[0] = make_ssa_name (narrow_vectype);
    8058            4 :   new_stmt = gimple_build_assign (vop[0], VIEW_CONVERT_EXPR,
    8059              :                                   build1 (VIEW_CONVERT_EXPR, narrow_vectype,
    8060              :                                           sub_res));
    8061            4 :   vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8062              : 
    8063              :   /* Emit:
    8064              : 
    8065              :        STAGE1 = DOT_PROD_EXPR <VOP[1], 64, VOP[2]>;
    8066              :        STAGE2 = DOT_PROD_EXPR <VOP[1], 64, STAGE1>;
    8067              :        STAGE3 = DOT_PROD_EXPR <SUB_RES, -128, STAGE2>;
    8068              : 
    8069              :      on the basis that x * y == (x - 128) * y + 64 * y + 64 * y
    8070              :      Doing the two 64 * y steps first allows more time to compute x.  */
    8071            4 :   tree stage1 = make_ssa_name (wide_vectype);
    8072            4 :   new_stmt = gimple_build_assign (stage1, DOT_PROD_EXPR,
    8073              :                                   vop[1], half_narrow, vop[2]);
    8074            4 :   vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8075              : 
    8076            4 :   tree stage2 = make_ssa_name (wide_vectype);
    8077            4 :   new_stmt = gimple_build_assign (stage2, DOT_PROD_EXPR,
    8078              :                                   vop[1], half_narrow, stage1);
    8079            4 :   vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8080              : 
    8081            4 :   tree stage3 = make_ssa_name (wide_vectype);
    8082            4 :   new_stmt = gimple_build_assign (stage3, DOT_PROD_EXPR,
    8083              :                                   vop[0], vop[1], stage2);
    8084            4 :   vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8085              : 
    8086              :   /* Convert STAGE3 to the reduction type.  */
    8087            4 :   return gimple_build_assign (vec_dest, CONVERT_EXPR, stage3);
    8088            4 : }
    8089              : 
    8090              : /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
    8091              :    value.  */
    8092              : 
    8093              : bool
    8094         2692 : vect_transform_reduction (loop_vec_info loop_vinfo,
    8095              :                           stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
    8096              :                           slp_tree slp_node)
    8097              : {
    8098         2692 :   tree vectype_out = SLP_TREE_VECTYPE (slp_node);
    8099         2692 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    8100              : 
    8101         2692 :   vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
    8102              : 
    8103         2692 :   if (nested_in_vect_loop_p (loop, stmt_info))
    8104              :     {
    8105            0 :       loop = loop->inner;
    8106            0 :       gcc_assert (VECT_REDUC_INFO_DEF_TYPE (reduc_info)
    8107              :                   == vect_double_reduction_def);
    8108              :     }
    8109              : 
    8110         2692 :   gimple_match_op op;
    8111         2692 :   if (!gimple_extract_op (stmt_info->stmt, &op))
    8112            0 :     gcc_unreachable ();
    8113              : 
    8114              :   /* All uses but the last are expected to be defined in the loop.
    8115              :      The last use is the reduction variable.  In case of nested cycle this
    8116              :      assumption is not true: we use reduc_index to record the index of the
    8117              :      reduction variable.  */
    8118         2692 :   int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
    8119         2692 :   tree vectype_in = SLP_TREE_VECTYPE (slp_node);
    8120         2692 :   if (lane_reducing_op_p (op.code))
    8121          281 :     vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]);
    8122              : 
    8123         2692 :   code_helper code = canonicalize_code (op.code, op.type);
    8124         2692 :   internal_fn cond_fn
    8125          484 :     = ((code.is_internal_fn ()
    8126          484 :         && internal_fn_mask_index ((internal_fn)code) != -1)
    8127         2692 :        ? (internal_fn)code : get_conditional_internal_fn (code, op.type));
    8128              : 
    8129         2692 :   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
    8130         2692 :   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
    8131         2692 :   bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
    8132              : 
    8133              :   /* Transform.  */
    8134         2692 :   tree new_temp = NULL_TREE;
    8135        18844 :   auto_vec<tree> vec_oprnds[3];
    8136              : 
    8137         2692 :   if (dump_enabled_p ())
    8138          807 :     dump_printf_loc (MSG_NOTE, vect_location, "transform reduction.\n");
    8139              : 
    8140              :   /* A binary COND_OP reduction must have the same definition and else
    8141              :      value. */
    8142         3176 :   bool cond_fn_p = code.is_internal_fn ()
    8143          484 :     && conditional_internal_fn_code (internal_fn (code)) != ERROR_MARK;
    8144          482 :   if (cond_fn_p)
    8145              :     {
    8146          482 :       gcc_assert (code == IFN_COND_ADD || code == IFN_COND_SUB
    8147              :                   || code == IFN_COND_MUL || code == IFN_COND_AND
    8148              :                   || code == IFN_COND_IOR || code == IFN_COND_XOR
    8149              :                   || code == IFN_COND_MIN || code == IFN_COND_MAX);
    8150          482 :       gcc_assert (op.num_ops == 4
    8151              :                   && (op.ops[reduc_index]
    8152              :                       == op.ops[internal_fn_else_index ((internal_fn) code)]));
    8153              :     }
    8154              : 
    8155         2692 :   bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
    8156              : 
    8157         2692 :   vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
    8158         2692 :   if (reduction_type == FOLD_LEFT_REDUCTION)
    8159              :     {
    8160          977 :       internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
    8161          977 :       gcc_assert (code.is_tree_code () || cond_fn_p);
    8162          977 :       return vectorize_fold_left_reduction
    8163          977 :           (loop_vinfo, stmt_info, gsi, slp_node,
    8164          977 :            code, reduc_fn, op.num_ops, vectype_in,
    8165          977 :            reduc_index, masks, lens);
    8166              :     }
    8167              : 
    8168         1715 :   bool single_defuse_cycle = VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info);
    8169         1715 :   bool lane_reducing = lane_reducing_op_p (code);
    8170         1434 :   gcc_assert (single_defuse_cycle || lane_reducing);
    8171              : 
    8172         1715 :   if (lane_reducing)
    8173              :     {
    8174              :       /* The last operand of lane-reducing op is for reduction.  */
    8175          281 :       gcc_assert (reduc_index == (int) op.num_ops - 1);
    8176              :     }
    8177              : 
    8178              :   /* Create the destination vector  */
    8179         1715 :   tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
    8180         1715 :   tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
    8181              : 
    8182              :   /* Get NCOPIES vector definitions for all operands except the reduction
    8183              :      definition.  */
    8184         1715 :   if (!cond_fn_p)
    8185              :     {
    8186         1262 :       gcc_assert (reduc_index >= 0 && reduc_index <= 2);
    8187         1262 :       vect_get_vec_defs (loop_vinfo, slp_node,
    8188         1262 :                          single_defuse_cycle && reduc_index == 0
    8189         1262 :                          ? NULL_TREE : op.ops[0], &vec_oprnds[0],
    8190         1262 :                          single_defuse_cycle && reduc_index == 1
    8191         1262 :                          ? NULL_TREE : op.ops[1], &vec_oprnds[1],
    8192         1262 :                          op.num_ops == 3
    8193          283 :                          && !(single_defuse_cycle && reduc_index == 2)
    8194         1369 :                          ? op.ops[2] : NULL_TREE, &vec_oprnds[2]);
    8195              :     }
    8196              :   else
    8197              :     {
    8198              :       /* For a conditional operation pass the truth type as mask
    8199              :          vectype.  */
    8200          453 :       gcc_assert (single_defuse_cycle
    8201              :                   && (reduc_index == 1 || reduc_index == 2));
    8202          453 :       vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
    8203              :                          &vec_oprnds[0],
    8204            2 :                          reduc_index == 1 ? NULL_TREE : op.ops[1],
    8205              :                          &vec_oprnds[1],
    8206          453 :                          reduc_index == 2 ? NULL_TREE : op.ops[2],
    8207              :                          &vec_oprnds[2]);
    8208              :     }
    8209              : 
    8210              :   /* For single def-use cycles get one copy of the vectorized reduction
    8211              :      definition.  */
    8212         3017 :   unsigned vec_in_num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
    8213         1715 :   if (single_defuse_cycle)
    8214              :     {
    8215         1608 :       vect_get_vec_defs (loop_vinfo, slp_node,
    8216         1608 :                          reduc_index == 0 ? op.ops[0] : NULL_TREE,
    8217              :                          &vec_oprnds[0],
    8218         1608 :                          reduc_index == 1 ? op.ops[1] : NULL_TREE,
    8219              :                          &vec_oprnds[1],
    8220         1608 :                          reduc_index == 2 ? op.ops[2] : NULL_TREE,
    8221              :                          &vec_oprnds[2]);
    8222              :     }
    8223          107 :   else if (lane_reducing)
    8224              :     {
    8225              :       /* For normal reduction, consistency between vectorized def/use is
    8226              :          naturally ensured when mapping from scalar statement.  But if lane-
    8227              :          reducing op is involved in reduction, thing would become somewhat
    8228              :          complicated in that the op's result and operand for accumulation are
    8229              :          limited to less lanes than other operands, which certainly causes
    8230              :          def/use mismatch on adjacent statements around the op if do not have
    8231              :          any kind of specific adjustment.  One approach is to refit lane-
    8232              :          reducing op in the way of introducing new trivial pass-through copies
    8233              :          to fix possible def/use gap, so as to make it behave like a normal op.
    8234              :          And vector reduction PHIs are always generated to the full extent, no
    8235              :          matter lane-reducing op exists or not.  If some copies or PHIs are
    8236              :          actually superfluous, they would be cleaned up by passes after
    8237              :          vectorization.  An example for single-lane slp, lane-reducing ops
    8238              :          with mixed input vectypes in a reduction chain, is given as below.
    8239              :          Similarly, this handling is applicable for multiple-lane slp as well.
    8240              : 
    8241              :            int sum = 1;
    8242              :            for (i)
    8243              :              {
    8244              :                sum += d0[i] * d1[i];      // dot-prod <vector(16) char>
    8245              :                sum += w[i];               // widen-sum <vector(16) char>
    8246              :                sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
    8247              :                sum += n[i];               // normal <vector(4) int>
    8248              :              }
    8249              : 
    8250              :          The vector size is 128-bit,vectorization factor is 16.  Reduction
    8251              :          statements would be transformed as:
    8252              : 
    8253              :            vector<4> int sum_v0 = { 0, 0, 0, 1 };
    8254              :            vector<4> int sum_v1 = { 0, 0, 0, 0 };
    8255              :            vector<4> int sum_v2 = { 0, 0, 0, 0 };
    8256              :            vector<4> int sum_v3 = { 0, 0, 0, 0 };
    8257              : 
    8258              :            for (i / 16)
    8259              :              {
    8260              :                sum_v0 = DOT_PROD (d0_v0[i: 0 ~ 15], d1_v0[i: 0 ~ 15], sum_v0);
    8261              :                sum_v1 = sum_v1;  // copy
    8262              :                sum_v2 = sum_v2;  // copy
    8263              :                sum_v3 = sum_v3;  // copy
    8264              : 
    8265              :                sum_v0 = sum_v0;  // copy
    8266              :                sum_v1 = WIDEN_SUM (w_v1[i: 0 ~ 15], sum_v1);
    8267              :                sum_v2 = sum_v2;  // copy
    8268              :                sum_v3 = sum_v3;  // copy
    8269              : 
    8270              :                sum_v0 = sum_v0;  // copy
    8271              :                sum_v1 = SAD (s0_v1[i: 0 ~ 7 ], s1_v1[i: 0 ~ 7 ], sum_v1);
    8272              :                sum_v2 = SAD (s0_v2[i: 8 ~ 15], s1_v2[i: 8 ~ 15], sum_v2);
    8273              :                sum_v3 = sum_v3;  // copy
    8274              : 
    8275              :                sum_v0 += n_v0[i: 0  ~ 3 ];
    8276              :                sum_v1 += n_v1[i: 4  ~ 7 ];
    8277              :                sum_v2 += n_v2[i: 8  ~ 11];
    8278              :                sum_v3 += n_v3[i: 12 ~ 15];
    8279              :              }
    8280              : 
    8281              :          Moreover, for a higher instruction parallelism in final vectorized
    8282              :          loop, it is considered to make those effective vector lane-reducing
    8283              :          ops be distributed evenly among all def-use cycles.  In the above
    8284              :          example, DOT_PROD, WIDEN_SUM and SADs are generated into disparate
    8285              :          cycles, instruction dependency among them could be eliminated.  */
    8286          107 :       unsigned effec_ncopies = vec_oprnds[0].length ();
    8287          107 :       unsigned total_ncopies = vec_oprnds[reduc_index].length ();
    8288              : 
    8289          107 :       gcc_assert (effec_ncopies <= total_ncopies);
    8290              : 
    8291          107 :       if (effec_ncopies < total_ncopies)
    8292              :         {
    8293          294 :           for (unsigned i = 0; i < op.num_ops - 1; i++)
    8294              :             {
    8295          392 :               gcc_assert (vec_oprnds[i].length () == effec_ncopies);
    8296          196 :               vec_oprnds[i].safe_grow_cleared (total_ncopies);
    8297              :             }
    8298              :         }
    8299              : 
    8300          107 :       tree reduc_vectype_in = vectype_in;
    8301          107 :       gcc_assert (reduc_vectype_in);
    8302              :     }
    8303              : 
    8304         1715 :   bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (slp_node);
    8305         1715 :   unsigned num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
    8306         1715 :   unsigned mask_index = 0;
    8307              : 
    8308         7484 :   for (unsigned i = 0; i < num; ++i)
    8309              :     {
    8310         5769 :       gimple *new_stmt;
    8311         5769 :       tree vop[3] = { vec_oprnds[0][i], vec_oprnds[1][i],
    8312         8076 :                       op.num_ops >= 3 ? vec_oprnds[2][i] : NULL_TREE };
    8313         5769 :       if (!vop[0] || !vop[1])
    8314              :         {
    8315          503 :           tree reduc_vop = vec_oprnds[reduc_index][i];
    8316              : 
    8317              :           /* If could not generate an effective vector statement for current
    8318              :              portion of reduction operand, insert a trivial copy to simply
    8319              :              handle over the operand to other dependent statements.  */
    8320          503 :           gcc_assert (reduc_vop);
    8321              : 
    8322          503 :           if (TREE_CODE (reduc_vop) == SSA_NAME
    8323          503 :               && !SSA_NAME_IS_DEFAULT_DEF (reduc_vop))
    8324          503 :             new_stmt = SSA_NAME_DEF_STMT (reduc_vop);
    8325              :           else
    8326              :             {
    8327            0 :               new_temp = make_ssa_name (vec_dest);
    8328            0 :               new_stmt = gimple_build_assign (new_temp, reduc_vop);
    8329            0 :               vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
    8330              :                                            gsi);
    8331              :             }
    8332              :         }
    8333         5266 :       else if (masked_loop_p && !mask_by_cond_expr)
    8334              :         {
    8335              :           /* No conditional ifns have been defined for lane-reducing op
    8336              :              yet.  */
    8337           20 :           gcc_assert (!lane_reducing);
    8338              : 
    8339           32 :           auto_vec<tree> args (op.num_ops + (cond_fn_p ? 0 : 2));
    8340           20 :           tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    8341              :                                           vec_in_num, vectype_in,
    8342           20 :                                           mask_index++);
    8343           20 :           if (cond_fn_p)
    8344              :             {
    8345            8 :               gcc_assert (internal_fn_mask_index (internal_fn (code)) == 0);
    8346            8 :               mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask),
    8347              :                                        mask, vop[0], gsi);
    8348              :             }
    8349           20 :           gcc_assert (internal_fn_mask_index (internal_fn (cond_fn)) == 0);
    8350           20 :           args.quick_push (mask);
    8351           20 :           for (unsigned j = cond_fn_p ? 1 : 0;
    8352          104 :                j < op.num_ops - (cond_fn_p ? 1 : 0); ++j)
    8353           44 :             args.quick_push (vop[j]);
    8354           40 :           gcc_assert (internal_fn_else_index (internal_fn (cond_fn))
    8355              :                       == (int) args.length ());
    8356           20 :           args.quick_push (vop[reduc_index]);
    8357           20 :           gcall *call = gimple_build_call_internal_vec (cond_fn, args);
    8358           20 :           new_temp = make_ssa_name (vec_dest, call);
    8359           20 :           gimple_call_set_lhs (call, new_temp);
    8360           20 :           gimple_call_set_nothrow (call, true);
    8361           20 :           vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi);
    8362           20 :           new_stmt = call;
    8363           20 :         }
    8364              :       else
    8365              :         {
    8366         5246 :           if (masked_loop_p && mask_by_cond_expr)
    8367              :             {
    8368            4 :               tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
    8369              :                                               vec_in_num, vectype_in,
    8370              :                                               mask_index++);
    8371            4 :               build_vect_cond_expr (code, vop, mask, gsi);
    8372              :             }
    8373              : 
    8374         5246 :           if (emulated_mixed_dot_prod)
    8375            4 :             new_stmt = vect_emulate_mixed_dot_prod (loop_vinfo, stmt_info, gsi,
    8376              :                                                     vec_dest, vop);
    8377              : 
    8378         6584 :           else if (code.is_internal_fn () && !cond_fn_p)
    8379            0 :             new_stmt = gimple_build_call_internal (internal_fn (code),
    8380              :                                                    op.num_ops,
    8381              :                                                    vop[0], vop[1], vop[2]);
    8382         6584 :           else if (code.is_internal_fn () && cond_fn_p)
    8383         1342 :             new_stmt = gimple_build_call_internal (internal_fn (code),
    8384              :                                                    op.num_ops,
    8385              :                                                    vop[0], vop[1], vop[2],
    8386              :                                                    vop[reduc_index]);
    8387              :           else
    8388         3900 :             new_stmt = gimple_build_assign (vec_dest, tree_code (op.code),
    8389              :                                             vop[0], vop[1], vop[2]);
    8390         5246 :           new_temp = make_ssa_name (vec_dest, new_stmt);
    8391         5246 :           gimple_set_lhs (new_stmt, new_temp);
    8392         5246 :           vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
    8393              :         }
    8394              : 
    8395         5769 :       if (single_defuse_cycle && i < num - 1)
    8396         3392 :         vec_oprnds[reduc_index].safe_push (gimple_get_lhs (new_stmt));
    8397              :       else
    8398         2377 :         slp_node->push_vec_def (new_stmt);
    8399              :     }
    8400              : 
    8401              :   return true;
    8402        10768 : }
    8403              : 
    8404              : /* Transform phase of a cycle PHI.  */
    8405              : 
    8406              : bool
    8407        23672 : vect_transform_cycle_phi (loop_vec_info loop_vinfo,
    8408              :                           stmt_vec_info stmt_info,
    8409              :                           slp_tree slp_node, slp_instance slp_node_instance)
    8410              : {
    8411        23672 :   tree vectype_out = SLP_TREE_VECTYPE (slp_node);
    8412        23672 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    8413        23672 :   int i;
    8414        23672 :   bool nested_cycle = false;
    8415        23672 :   int vec_num;
    8416              : 
    8417        23672 :   if (nested_in_vect_loop_p (loop, stmt_info))
    8418              :     {
    8419        23672 :       loop = loop->inner;
    8420        23672 :       nested_cycle = true;
    8421              :     }
    8422              : 
    8423        23672 :   vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
    8424        23672 :   if (reduc_info
    8425        23089 :       && (VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
    8426        23089 :           || VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION))
    8427              :     /* Leave the scalar phi in place.  */
    8428              :     return true;
    8429              : 
    8430        22112 :   if (reduc_info && reduc_info->is_reduc_chain && dump_enabled_p ())
    8431          133 :     dump_printf_loc (MSG_NOTE, vect_location,
    8432              :                      "vectorizing a reduction chain\n");
    8433              : 
    8434        22695 :   vec_num = vect_get_num_copies (loop_vinfo, slp_node);
    8435              : 
    8436              :   /* Check whether we should use a single PHI node and accumulate
    8437              :      vectors to one before the backedge.  */
    8438        22695 :   if (reduc_info && VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info))
    8439        22695 :     vec_num = 1;
    8440              : 
    8441              :   /* Create the destination vector  */
    8442        22695 :   gphi *phi = as_a <gphi *> (stmt_info->stmt);
    8443        22695 :   tree vec_dest = vect_create_destination_var (gimple_phi_result (phi),
    8444              :                                                vectype_out);
    8445              : 
    8446              :   /* Get the loop-entry arguments.  */
    8447        22695 :   auto_vec<tree> vec_initial_defs;
    8448        22695 :   vec_initial_defs.reserve (vec_num);
    8449              :   /* Optimize: if initial_def is for REDUC_MAX smaller than the base
    8450              :      and we can't use zero for induc_val, use initial_def.  Similarly
    8451              :      for REDUC_MIN and initial_def larger than the base.  */
    8452        22695 :   if (reduc_info
    8453        22112 :       && VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
    8454              :     {
    8455           62 :       gcc_assert (SLP_TREE_LANES (slp_node) == 1);
    8456           62 :       tree initial_def = vect_phi_initial_value (phi);
    8457           62 :       VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).safe_push (initial_def);
    8458           62 :       tree induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
    8459           62 :       if (TREE_CODE (initial_def) == INTEGER_CST
    8460           60 :           && !integer_zerop (induc_val)
    8461          122 :           && ((VECT_REDUC_INFO_CODE (reduc_info) == MAX_EXPR
    8462           42 :                && tree_int_cst_lt (initial_def, induc_val))
    8463           58 :               || (VECT_REDUC_INFO_CODE (reduc_info) == MIN_EXPR
    8464           18 :                   && tree_int_cst_lt (induc_val, initial_def))))
    8465              :         {
    8466            2 :           induc_val = initial_def;
    8467              :           /* Communicate we used the initial_def to epilouge
    8468              :              generation.  */
    8469            2 :           VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
    8470              :         }
    8471           62 :       vec_initial_defs.quick_push
    8472           62 :         (build_vector_from_val (vectype_out, induc_val));
    8473           62 :     }
    8474        22633 :   else if (nested_cycle)
    8475              :     {
    8476          663 :       unsigned phi_idx = loop_preheader_edge (loop)->dest_idx;
    8477          663 :       vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[phi_idx],
    8478              :                          &vec_initial_defs);
    8479              :     }
    8480              :   else
    8481              :     {
    8482        21970 :       gcc_assert (slp_node == slp_node_instance->reduc_phis);
    8483        21970 :       vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
    8484        21970 :       vec<stmt_vec_info> &stmts = SLP_TREE_SCALAR_STMTS (slp_node);
    8485              : 
    8486        21970 :       unsigned int num_phis = stmts.length ();
    8487        21970 :       if (reduc_info->is_reduc_chain)
    8488          215 :         num_phis = 1;
    8489        21970 :       initial_values.reserve (num_phis);
    8490        66387 :       for (unsigned int i = 0; i < num_phis; ++i)
    8491              :         {
    8492        22447 :           gphi *this_phi = as_a<gphi *> (stmts[i]->stmt);
    8493        22447 :           initial_values.quick_push (vect_phi_initial_value (this_phi));
    8494              :         }
    8495        21970 :       tree neutral_op = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
    8496        21970 :       if (vec_num == 1
    8497        21970 :           && vect_find_reusable_accumulator (loop_vinfo,
    8498              :                                              reduc_info, vectype_out))
    8499              :         ;
    8500              :       /* Try to simplify the vector initialization by applying an
    8501              :          adjustment after the reduction has been performed.  This
    8502              :          can also break a critical path but on the other hand
    8503              :          requires to keep the initial value live across the loop.  */
    8504        18033 :       else if (neutral_op
    8505        17444 :                && initial_values.length () == 1
    8506        17235 :                && STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
    8507        35197 :                && !operand_equal_p (neutral_op, initial_values[0]))
    8508              :         {
    8509        12139 :           VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info)
    8510        12139 :             = initial_values[0];
    8511        12139 :           initial_values[0] = neutral_op;
    8512              :         }
    8513        21970 :       if (!VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
    8514         3937 :           || loop_vinfo->main_loop_edge)
    8515        43642 :         get_initial_defs_for_reduction (loop_vinfo, reduc_info, vectype_out,
    8516              :                                         &vec_initial_defs, vec_num,
    8517              :                                         stmts.length (), neutral_op);
    8518              :     }
    8519              : 
    8520        22695 :   if (reduc_info)
    8521        22112 :   if (auto *accumulator = VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
    8522              :     {
    8523         3937 :       tree def = accumulator->reduc_input;
    8524         3937 :       if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
    8525              :         {
    8526         3934 :           unsigned int nreduc;
    8527         7868 :           bool res = constant_multiple_p (TYPE_VECTOR_SUBPARTS
    8528         3934 :                                             (TREE_TYPE (def)),
    8529         3934 :                                           TYPE_VECTOR_SUBPARTS (vectype_out),
    8530              :                                           &nreduc);
    8531            0 :           gcc_assert (res);
    8532         3934 :           gimple_seq stmts = NULL;
    8533              :           /* Reduce the single vector to a smaller one.  */
    8534         3934 :           if (nreduc != 1)
    8535              :             {
    8536              :               /* Perform the reduction in the appropriate type.  */
    8537         3934 :               tree rvectype = vectype_out;
    8538         3934 :               if (!useless_type_conversion_p (TREE_TYPE (vectype_out),
    8539         3934 :                                               TREE_TYPE (TREE_TYPE (def))))
    8540          235 :                 rvectype = build_vector_type (TREE_TYPE (TREE_TYPE (def)),
    8541              :                                               TYPE_VECTOR_SUBPARTS
    8542          470 :                                                 (vectype_out));
    8543         3934 :               def = vect_create_partial_epilog (def, rvectype,
    8544              :                                                 VECT_REDUC_INFO_CODE
    8545              :                                                   (reduc_info),
    8546              :                                                 &stmts);
    8547              :             }
    8548              :           /* The epilogue loop might use a different vector mode, like
    8549              :              VNx2DI vs. V2DI.  */
    8550         3934 :           if (TYPE_MODE (vectype_out) != TYPE_MODE (TREE_TYPE (def)))
    8551              :             {
    8552            0 :               tree reduc_type = build_vector_type_for_mode
    8553            0 :                 (TREE_TYPE (TREE_TYPE (def)), TYPE_MODE (vectype_out));
    8554            0 :               def = gimple_convert (&stmts, reduc_type, def);
    8555              :             }
    8556              :           /* Adjust the input so we pick up the partially reduced value
    8557              :              for the skip edge in vect_create_epilog_for_reduction.  */
    8558         3934 :           accumulator->reduc_input = def;
    8559              :           /* And the reduction could be carried out using a different sign.  */
    8560         3934 :           if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
    8561          235 :             def = gimple_convert (&stmts, vectype_out, def);
    8562         3934 :           edge e;
    8563         3934 :           if ((e = loop_vinfo->main_loop_edge)
    8564         3934 :               || (e = loop_vinfo->skip_this_loop_edge))
    8565              :             {
    8566              :               /* While we'd like to insert on the edge this will split
    8567              :                  blocks and disturb bookkeeping, we also will eventually
    8568              :                  need this on the skip edge.  Rely on sinking to
    8569              :                  fixup optimal placement and insert in the pred.  */
    8570         3785 :               gimple_stmt_iterator gsi = gsi_last_bb (e->src);
    8571              :               /* Insert before a cond that eventually skips the
    8572              :                  epilogue.  */
    8573         3785 :               if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
    8574         3768 :                 gsi_prev (&gsi);
    8575         3785 :               gsi_insert_seq_after (&gsi, stmts, GSI_CONTINUE_LINKING);
    8576              :             }
    8577              :           else
    8578          149 :             gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop),
    8579              :                                               stmts);
    8580              :         }
    8581         3937 :       if (loop_vinfo->main_loop_edge)
    8582         3788 :         vec_initial_defs[0]
    8583         3788 :           = vect_get_main_loop_result (loop_vinfo, def,
    8584         3788 :                                        vec_initial_defs[0]);
    8585              :       else
    8586          149 :         vec_initial_defs.safe_push (def);
    8587              :     }
    8588              : 
    8589              :   /* Generate the reduction PHIs upfront.  */
    8590        47244 :   for (i = 0; i < vec_num; i++)
    8591              :     {
    8592        24549 :       tree vec_init_def = vec_initial_defs[i];
    8593              :       /* Create the reduction-phi that defines the reduction
    8594              :          operand.  */
    8595        24549 :       gphi *new_phi = create_phi_node (vec_dest, loop->header);
    8596        24549 :       add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
    8597              :                    UNKNOWN_LOCATION);
    8598              : 
    8599              :       /* The loop-latch arg is set in epilogue processing.  */
    8600              : 
    8601        24549 :       slp_node->push_vec_def (new_phi);
    8602              :     }
    8603              : 
    8604        22695 :   return true;
    8605        22695 : }
    8606              : 
    8607              : /* Vectorizes LC PHIs.  */
    8608              : 
    8609              : bool
    8610       206990 : vectorizable_lc_phi (loop_vec_info loop_vinfo,
    8611              :                      stmt_vec_info stmt_info,
    8612              :                      slp_tree slp_node)
    8613              : {
    8614       206990 :   if (!loop_vinfo
    8615       206990 :       || !is_a <gphi *> (stmt_info->stmt)
    8616       251863 :       || gimple_phi_num_args (stmt_info->stmt) != 1)
    8617              :     return false;
    8618              : 
    8619          805 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
    8620            0 :       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
    8621              :     return false;
    8622              : 
    8623              :   /* Deal with copies from externs or constants that disguise as
    8624              :      loop-closed PHI nodes (PR97886).  */
    8625          805 :   if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
    8626              :                                          SLP_TREE_VECTYPE (slp_node)))
    8627              :     {
    8628            0 :       if (dump_enabled_p ())
    8629            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8630              :                          "incompatible vector types for invariants\n");
    8631              :       return false;
    8632              :     }
    8633              : 
    8634              :   /* ???  This can happen with data vs. mask uses of boolean.  */
    8635          805 :   if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
    8636          805 :                                   SLP_TREE_VECTYPE
    8637              :                                     (SLP_TREE_CHILDREN (slp_node)[0])))
    8638              :     {
    8639            0 :       if (dump_enabled_p ())
    8640            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8641              :                          "missed mask promotion\n");
    8642              :       return false;
    8643              :     }
    8644              : 
    8645          805 :   SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
    8646          805 :   return true;
    8647              : }
    8648              : 
    8649              : bool
    8650          489 : vect_transform_lc_phi (loop_vec_info loop_vinfo,
    8651              :                        stmt_vec_info stmt_info,
    8652              :                        slp_tree slp_node)
    8653              : {
    8654              : 
    8655          489 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    8656          489 :   tree scalar_dest = gimple_phi_result (stmt_info->stmt);
    8657          489 :   basic_block bb = gimple_bb (stmt_info->stmt);
    8658          489 :   edge e = single_pred_edge (bb);
    8659          489 :   tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
    8660          489 :   auto_vec<tree> vec_oprnds;
    8661          489 :   vect_get_vec_defs (loop_vinfo, slp_node, true, &vec_oprnds);
    8662         1577 :   for (unsigned i = 0; i < vec_oprnds.length (); i++)
    8663              :     {
    8664              :       /* Create the vectorized LC PHI node.  */
    8665          599 :       gphi *new_phi = create_phi_node (vec_dest, bb);
    8666          599 :       add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
    8667          599 :       slp_node->push_vec_def (new_phi);
    8668              :     }
    8669              : 
    8670          489 :   return true;
    8671          489 : }
    8672              : 
    8673              : /* Vectorizes PHIs.  */
    8674              : 
    8675              : bool
    8676       153900 : vectorizable_phi (bb_vec_info vinfo,
    8677              :                   stmt_vec_info stmt_info,
    8678              :                   slp_tree slp_node, stmt_vector_for_cost *cost_vec)
    8679              : {
    8680       153900 :   if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
    8681              :     return false;
    8682              : 
    8683        73823 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
    8684              :     return false;
    8685              : 
    8686        73823 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    8687              : 
    8688        73823 :   if (cost_vec) /* transformation not required.  */
    8689              :     {
    8690              :       slp_tree child;
    8691              :       unsigned i;
    8692       206347 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), i, child)
    8693       145286 :         if (!child)
    8694              :           {
    8695            0 :             if (dump_enabled_p ())
    8696            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8697              :                                "PHI node with unvectorized backedge def\n");
    8698              :             return false;
    8699              :           }
    8700       145286 :         else if (!vect_maybe_update_slp_op_vectype (child, vectype))
    8701              :           {
    8702           26 :             if (dump_enabled_p ())
    8703            2 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8704              :                                "incompatible vector types for invariants\n");
    8705              :             return false;
    8706              :           }
    8707       145260 :         else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
    8708       145260 :                  && !useless_type_conversion_p (vectype,
    8709              :                                                 SLP_TREE_VECTYPE (child)))
    8710              :           {
    8711              :             /* With bools we can have mask and non-mask precision vectors
    8712              :                or different non-mask precisions.  while pattern recog is
    8713              :                supposed to guarantee consistency here bugs in it can cause
    8714              :                mismatches (PR103489 and PR103800 for example).
    8715              :                Deal with them here instead of ICEing later.  */
    8716           18 :             if (dump_enabled_p ())
    8717            8 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8718              :                                "incompatible vector type setup from "
    8719              :                                "bool pattern detection\n");
    8720              :             return false;
    8721              :           }
    8722              : 
    8723              :       /* For single-argument PHIs assume coalescing which means zero cost
    8724              :          for the scalar and the vector PHIs.  This avoids artificially
    8725              :          favoring the vector path (but may pessimize it in some cases).  */
    8726        61061 :       if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
    8727        55307 :         record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, slp_node),
    8728              :                           vector_stmt, slp_node, vectype, 0, vect_body);
    8729        61061 :       SLP_TREE_TYPE (slp_node) = phi_info_type;
    8730        61061 :       return true;
    8731              :     }
    8732              : 
    8733        12718 :   tree scalar_dest = gimple_phi_result (stmt_info->stmt);
    8734        12718 :   basic_block bb = gimple_bb (stmt_info->stmt);
    8735        12718 :   tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
    8736        12718 :   auto_vec<gphi *> new_phis;
    8737        44889 :   for (unsigned i = 0; i < gimple_phi_num_args (stmt_info->stmt); ++i)
    8738              :     {
    8739        32171 :       slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
    8740              : 
    8741              :       /* Skip not yet vectorized defs.  */
    8742        32760 :       if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
    8743        32171 :           && SLP_TREE_VEC_DEFS (child).is_empty ())
    8744          589 :         continue;
    8745              : 
    8746        31582 :       auto_vec<tree> vec_oprnds;
    8747        31582 :       vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[i], &vec_oprnds);
    8748        31582 :       if (!new_phis.exists ())
    8749              :         {
    8750        12718 :           new_phis.create (vec_oprnds.length ());
    8751        39433 :           for (unsigned j = 0; j < vec_oprnds.length (); j++)
    8752              :             {
    8753              :               /* Create the vectorized LC PHI node.  */
    8754        13997 :               new_phis.quick_push (create_phi_node (vec_dest, bb));
    8755        13997 :               slp_node->push_vec_def (new_phis[j]);
    8756              :             }
    8757              :         }
    8758        31582 :       edge e = gimple_phi_arg_edge (as_a <gphi *> (stmt_info->stmt), i);
    8759        67447 :       for (unsigned j = 0; j < vec_oprnds.length (); j++)
    8760        35865 :         add_phi_arg (new_phis[j], vec_oprnds[j], e, UNKNOWN_LOCATION);
    8761        31582 :     }
    8762              :   /* We should have at least one already vectorized child.  */
    8763        12718 :   gcc_assert (new_phis.exists ());
    8764              : 
    8765        12718 :   return true;
    8766        12718 : }
    8767              : 
    8768              : /* Vectorizes first order recurrences.  An overview of the transformation
    8769              :    is described below. Suppose we have the following loop.
    8770              : 
    8771              :      int t = 0;
    8772              :      for (int i = 0; i < n; ++i)
    8773              :        {
    8774              :          b[i] = a[i] - t;
    8775              :          t = a[i];
    8776              :        }
    8777              : 
    8778              :    There is a first-order recurrence on 'a'. For this loop, the scalar IR
    8779              :    looks (simplified) like:
    8780              : 
    8781              :     scalar.preheader:
    8782              :       init = 0;
    8783              : 
    8784              :     scalar.body:
    8785              :       i = PHI <0(scalar.preheader), i+1(scalar.body)>
    8786              :       _2 = PHI <(init(scalar.preheader), <_1(scalar.body)>
    8787              :       _1 = a[i]
    8788              :       b[i] = _1 - _2
    8789              :       if (i < n) goto scalar.body
    8790              : 
    8791              :    In this example, _2 is a recurrence because it's value depends on the
    8792              :    previous iteration.  We vectorize this as (VF = 4)
    8793              : 
    8794              :     vector.preheader:
    8795              :       vect_init = vect_cst(..., ..., ..., 0)
    8796              : 
    8797              :     vector.body
    8798              :       i = PHI <0(vector.preheader), i+4(vector.body)>
    8799              :       vect_1 = PHI <vect_init(vector.preheader), v2(vector.body)>
    8800              :       vect_2 = a[i, i+1, i+2, i+3];
    8801              :       vect_3 = vec_perm (vect_1, vect_2, { 3, 4, 5, 6 })
    8802              :       b[i, i+1, i+2, i+3] = vect_2 - vect_3
    8803              :       if (..) goto vector.body
    8804              : 
    8805              :    In this function, vectorizable_recurr, we code generate both the
    8806              :    vector PHI node and the permute since those together compute the
    8807              :    vectorized value of the scalar PHI.  We do not yet have the
    8808              :    backedge value to fill in there nor into the vec_perm.  Those
    8809              :    are filled in vect_schedule_scc.
    8810              : 
    8811              :    TODO:  Since the scalar loop does not have a use of the recurrence
    8812              :    outside of the loop the natural way to implement peeling via
    8813              :    vectorizing the live value doesn't work.  For now peeling of loops
    8814              :    with a recurrence is not implemented.  For SLP the supported cases
    8815              :    are restricted to those requiring a single vector recurrence PHI.  */
    8816              : 
    8817              : bool
    8818       206228 : vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
    8819              :                      slp_tree slp_node, stmt_vector_for_cost *cost_vec)
    8820              : {
    8821       206228 :   if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
    8822              :     return false;
    8823              : 
    8824        44111 :   gphi *phi = as_a<gphi *> (stmt_info->stmt);
    8825              : 
    8826              :   /* So far we only support first-order recurrence auto-vectorization.  */
    8827        44111 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
    8828              :     return false;
    8829              : 
    8830          424 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    8831          424 :   unsigned ncopies = vect_get_num_copies (loop_vinfo, slp_node);
    8832          424 :   poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    8833          424 :   unsigned dist = SLP_TREE_LANES (slp_node);
    8834              :   /* We need to be able to make progress with a single vector.  */
    8835          424 :   if (maybe_gt (dist * 2, nunits))
    8836              :     {
    8837            0 :       if (dump_enabled_p ())
    8838            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8839              :                          "first order recurrence exceeds half of "
    8840              :                          "a vector\n");
    8841              :       return false;
    8842              :     }
    8843              : 
    8844              :   /* We need to be able to build a { ..., a, b } init vector with
    8845              :      dist number of distinct trailing values.  Always possible
    8846              :      when dist == 1 or when nunits is constant or when the initializations
    8847              :      are uniform.  */
    8848          424 :   tree uniform_initval = NULL_TREE;
    8849          424 :   edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
    8850         1720 :   for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
    8851              :     {
    8852          460 :       gphi *phi = as_a <gphi *> (s->stmt);
    8853          460 :       if (! uniform_initval)
    8854          424 :         uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
    8855           36 :       else if (! operand_equal_p (uniform_initval,
    8856           36 :                                   PHI_ARG_DEF_FROM_EDGE (phi, pe)))
    8857              :         {
    8858              :           uniform_initval = NULL_TREE;
    8859              :           break;
    8860              :         }
    8861              :     }
    8862          424 :   if (!uniform_initval && !nunits.is_constant ())
    8863              :     {
    8864              :       if (dump_enabled_p ())
    8865              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8866              :                          "cannot build initialization vector for "
    8867              :                          "first order recurrence\n");
    8868              :       return false;
    8869              :     }
    8870              : 
    8871              :   /* First-order recurrence autovectorization needs to handle permutation
    8872              :      with indices = [nunits-1, nunits, nunits+1, ...].  */
    8873          424 :   vec_perm_builder sel (nunits, 1, 3);
    8874         1696 :   for (int i = 0; i < 3; ++i)
    8875         1272 :     sel.quick_push (nunits - dist + i);
    8876          424 :   vec_perm_indices indices (sel, 2, nunits);
    8877              : 
    8878          424 :   if (cost_vec) /* transformation not required.  */
    8879              :     {
    8880          381 :       if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
    8881              :                                  indices))
    8882              :         return false;
    8883              : 
    8884              :       /* We eventually need to set a vector type on invariant
    8885              :          arguments.  */
    8886              :       unsigned j;
    8887              :       slp_tree child;
    8888          807 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
    8889          538 :         if (!vect_maybe_update_slp_op_vectype (child, vectype))
    8890              :           {
    8891            0 :             if (dump_enabled_p ())
    8892            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    8893              :                                "incompatible vector types for "
    8894              :                                "invariants\n");
    8895              :             return false;
    8896              :           }
    8897              : 
    8898              :       /* Verify we have set up compatible types.  */
    8899          269 :       edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
    8900          269 :       slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
    8901          269 :       tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
    8902          269 :       if (!types_compatible_p (latch_vectype, vectype))
    8903              :         return false;
    8904              : 
    8905              :       /* The recurrence costs the initialization vector and one permute
    8906              :          for each copy.  With SLP the prologue value is explicitly
    8907              :          represented and costed separately.  */
    8908          269 :       unsigned prologue_cost = 0;
    8909          269 :       unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
    8910              :                                                slp_node, 0, vect_body);
    8911          269 :       if (dump_enabled_p ())
    8912           51 :         dump_printf_loc (MSG_NOTE, vect_location,
    8913              :                          "vectorizable_recurr: inside_cost = %d, "
    8914              :                          "prologue_cost = %d .\n", inside_cost,
    8915              :                          prologue_cost);
    8916              : 
    8917          269 :       SLP_TREE_TYPE (slp_node) = recurr_info_type;
    8918          269 :       return true;
    8919              :     }
    8920              : 
    8921           43 :   tree vec_init;
    8922           43 :   if (! uniform_initval)
    8923              :     {
    8924            6 :       vec<constructor_elt, va_gc> *v = NULL;
    8925            6 :       vec_alloc (v, nunits.to_constant ());
    8926           39 :       for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
    8927           27 :         CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
    8928              :                                 build_zero_cst (TREE_TYPE (vectype)));
    8929           39 :       for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
    8930              :         {
    8931           21 :           gphi *phi = as_a <gphi *> (s->stmt);
    8932           21 :           tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
    8933           21 :           if (!useless_type_conversion_p (TREE_TYPE (vectype),
    8934           21 :                                           TREE_TYPE (preheader)))
    8935              :             {
    8936            0 :               gimple_seq stmts = NULL;
    8937            0 :               preheader = gimple_convert (&stmts,
    8938            0 :                                           TREE_TYPE (vectype), preheader);
    8939            0 :               gsi_insert_seq_on_edge_immediate (pe, stmts);
    8940              :             }
    8941           21 :           CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
    8942              :         }
    8943            6 :       vec_init = build_constructor (vectype, v);
    8944              :     }
    8945              :   else
    8946              :     vec_init = uniform_initval;
    8947           43 :   vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
    8948              : 
    8949              :   /* Create the vectorized first-order PHI node.  */
    8950           43 :   tree vec_dest = vect_get_new_vect_var (vectype,
    8951              :                                          vect_simple_var, "vec_recur_");
    8952           43 :   basic_block bb = gimple_bb (phi);
    8953           43 :   gphi *new_phi = create_phi_node (vec_dest, bb);
    8954           43 :   add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
    8955              : 
    8956              :   /* Insert shuffles the first-order recurrence autovectorization.
    8957              :        result = VEC_PERM <vec_recur, vect_1, index[nunits-1, nunits, ...]>.  */
    8958           43 :   tree perm = vect_gen_perm_mask_checked (vectype, indices);
    8959              : 
    8960              :   /* Insert the required permute after the latch definition.  The
    8961              :      second and later operands are tentative and will be updated when we have
    8962              :      vectorized the latch definition.  */
    8963           43 :   edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
    8964           43 :   gimple *latch_def = SSA_NAME_DEF_STMT (PHI_ARG_DEF_FROM_EDGE (phi, le));
    8965           43 :   gimple_stmt_iterator gsi2 = gsi_for_stmt (latch_def);
    8966           49 :   do
    8967              :     {
    8968           49 :       gsi_next (&gsi2);
    8969              :     }
    8970              :   /* Skip inserted vectorized stmts for the latch definition.  We have to
    8971              :      insert after those.  */
    8972           92 :   while (gsi_stmt (gsi2) && gimple_uid (gsi_stmt (gsi2)) == 0);
    8973              : 
    8974          123 :   for (unsigned i = 0; i < ncopies; ++i)
    8975              :     {
    8976           80 :       vec_dest = make_ssa_name (vectype);
    8977           80 :       gassign *vperm
    8978          123 :           = gimple_build_assign (vec_dest, VEC_PERM_EXPR,
    8979           43 :                                  i == 0 ? gimple_phi_result (new_phi) : NULL,
    8980              :                                  NULL, perm);
    8981           80 :       vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
    8982              : 
    8983           80 :       slp_node->push_vec_def (vperm);
    8984              :     }
    8985              : 
    8986              :   return true;
    8987          424 : }
    8988              : 
    8989              : /* Return true if VECTYPE represents a vector that requires lowering
    8990              :    by the vector lowering pass.  */
    8991              : 
    8992              : bool
    8993       680116 : vect_emulated_vector_p (tree vectype)
    8994              : {
    8995      1360232 :   return (!VECTOR_MODE_P (TYPE_MODE (vectype))
    8996       684246 :           && (!VECTOR_BOOLEAN_TYPE_P (vectype)
    8997         4112 :               || TYPE_PRECISION (TREE_TYPE (vectype)) != 1));
    8998              : }
    8999              : 
    9000              : /* Return true if we can emulate CODE on an integer mode representation
    9001              :    of a vector.  */
    9002              : 
    9003              : bool
    9004        12364 : vect_can_vectorize_without_simd_p (tree_code code)
    9005              : {
    9006        12364 :   switch (code)
    9007              :     {
    9008              :     case PLUS_EXPR:
    9009              :     case MINUS_EXPR:
    9010              :     case NEGATE_EXPR:
    9011              :     case BIT_AND_EXPR:
    9012              :     case BIT_IOR_EXPR:
    9013              :     case BIT_XOR_EXPR:
    9014              :     case BIT_NOT_EXPR:
    9015              :       return true;
    9016              : 
    9017        11754 :     default:
    9018        11754 :       return false;
    9019              :     }
    9020              : }
    9021              : 
    9022              : /* Likewise, but taking a code_helper.  */
    9023              : 
    9024              : bool
    9025         1003 : vect_can_vectorize_without_simd_p (code_helper code)
    9026              : {
    9027         1003 :   return (code.is_tree_code ()
    9028         1003 :           && vect_can_vectorize_without_simd_p (tree_code (code)));
    9029              : }
    9030              : 
    9031              : /* Create vector init for vectorized iv.  */
    9032              : static tree
    9033          932 : vect_create_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
    9034              :                                tree step_expr, poly_uint64 nunits,
    9035              :                                tree vectype,
    9036              :                                enum vect_induction_op_type induction_type)
    9037              : {
    9038          932 :   unsigned HOST_WIDE_INT const_nunits;
    9039          932 :   tree vec_shift, vec_init, new_name;
    9040          932 :   unsigned i;
    9041          932 :   tree itype = TREE_TYPE (vectype);
    9042              : 
    9043              :   /* iv_loop is the loop to be vectorized. Create:
    9044              :      vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr).  */
    9045          932 :   new_name = gimple_convert (stmts, itype, init_expr);
    9046          932 :   switch (induction_type)
    9047              :     {
    9048           21 :     case vect_step_op_shr:
    9049           21 :     case vect_step_op_shl:
    9050              :       /* Build the Initial value from shift_expr.  */
    9051           21 :       vec_init = gimple_build_vector_from_val (stmts,
    9052              :                                                vectype,
    9053              :                                                new_name);
    9054           21 :       vec_shift = gimple_build (stmts, VEC_SERIES_EXPR, vectype,
    9055              :                                 build_zero_cst (itype), step_expr);
    9056           21 :       vec_init = gimple_build (stmts,
    9057              :                                (induction_type == vect_step_op_shr
    9058              :                                 ? RSHIFT_EXPR : LSHIFT_EXPR),
    9059              :                                vectype, vec_init, vec_shift);
    9060           21 :       break;
    9061              : 
    9062          833 :     case vect_step_op_neg:
    9063          833 :       {
    9064          833 :         vec_init = gimple_build_vector_from_val (stmts,
    9065              :                                                  vectype,
    9066              :                                                  new_name);
    9067          833 :         tree vec_neg = gimple_build (stmts, NEGATE_EXPR,
    9068              :                                      vectype, vec_init);
    9069              :         /* The encoding has 2 interleaved stepped patterns.  */
    9070          833 :         vec_perm_builder sel (nunits, 2, 3);
    9071          833 :         sel.quick_grow (6);
    9072         4165 :         for (i = 0; i < 3; i++)
    9073              :           {
    9074         2499 :             sel[2 * i] = i;
    9075         2499 :             sel[2 * i + 1] = i + nunits;
    9076              :           }
    9077          833 :         vec_perm_indices indices (sel, 2, nunits);
    9078              :         /* Don't use vect_gen_perm_mask_checked since can_vec_perm_const_p may
    9079              :            fail when vec_init is const vector. In that situation vec_perm is not
    9080              :            really needed.  */
    9081          833 :         tree perm_mask_even
    9082          833 :           = vect_gen_perm_mask_any (vectype, indices);
    9083          833 :         vec_init = gimple_build (stmts, VEC_PERM_EXPR,
    9084              :                                  vectype,
    9085              :                                  vec_init, vec_neg,
    9086              :                                  perm_mask_even);
    9087          833 :       }
    9088          833 :       break;
    9089              : 
    9090           78 :     case vect_step_op_mul:
    9091           78 :       {
    9092              :         /* Use unsigned mult to avoid UD integer overflow.  */
    9093           78 :         gcc_assert (nunits.is_constant (&const_nunits));
    9094           78 :         tree utype = unsigned_type_for (itype);
    9095           78 :         tree uvectype = build_vector_type (utype,
    9096           78 :                                            TYPE_VECTOR_SUBPARTS (vectype));
    9097           78 :         new_name = gimple_convert (stmts, utype, new_name);
    9098           78 :         vec_init = gimple_build_vector_from_val (stmts,
    9099              :                                                  uvectype,
    9100              :                                                  new_name);
    9101           78 :         tree_vector_builder elts (uvectype, const_nunits, 1);
    9102           78 :         tree elt_step = build_one_cst (utype);
    9103              : 
    9104           78 :         elts.quick_push (elt_step);
    9105          670 :         for (i = 1; i < const_nunits; i++)
    9106              :           {
    9107              :             /* Create: new_name_i = new_name + step_expr.  */
    9108          514 :             elt_step = gimple_build (stmts, MULT_EXPR,
    9109              :                                      utype, elt_step, step_expr);
    9110          514 :             elts.quick_push (elt_step);
    9111              :           }
    9112              :         /* Create a vector from [new_name_0, new_name_1, ...,
    9113              :            new_name_nunits-1].  */
    9114           78 :         tree vec_mul = gimple_build_vector (stmts, &elts);
    9115           78 :         vec_init = gimple_build (stmts, MULT_EXPR, uvectype,
    9116              :                                  vec_init, vec_mul);
    9117           78 :         vec_init = gimple_convert (stmts, vectype, vec_init);
    9118           78 :       }
    9119           78 :       break;
    9120              : 
    9121            0 :     default:
    9122            0 :       gcc_unreachable ();
    9123              :     }
    9124              : 
    9125          932 :   return vec_init;
    9126              : }
    9127              : 
    9128              : /* Peel init_expr by skip_niter for induction_type.  */
    9129              : tree
    9130           83 : vect_peel_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
    9131              :                              tree skip_niters, tree step_expr,
    9132              :                              enum vect_induction_op_type induction_type,
    9133              :                              bool early_exit_p)
    9134              : {
    9135           83 :   gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST || early_exit_p);
    9136           83 :   tree type = TREE_TYPE (init_expr);
    9137           83 :   unsigned prec = TYPE_PRECISION (type);
    9138           83 :   switch (induction_type)
    9139              :     {
    9140              :     /* neg inductions are typically not used for loop termination conditions but
    9141              :        are typically implemented as b = -b.  That is every scalar iteration b is
    9142              :        negated.  That means that for the initial value of b we will have to
    9143              :        determine whether the number of skipped iteration is a multiple of 2
    9144              :        because every 2 scalar iterations we are back at "b".  */
    9145            0 :     case vect_step_op_neg:
    9146              :       /* For early exits the neg induction will always be the same value at the
    9147              :          start of the iteration.  */
    9148            0 :       if (early_exit_p)
    9149              :         break;
    9150              : 
    9151            0 :       if (TREE_INT_CST_LOW (skip_niters) % 2)
    9152            0 :         init_expr = gimple_build (stmts, NEGATE_EXPR, type, init_expr);
    9153              :       /* else no change.  */
    9154              :       break;
    9155              : 
    9156           12 :     case vect_step_op_shr:
    9157           12 :     case vect_step_op_shl:
    9158              :       /* When shift count >= precision, need to avoid UD.
    9159              :          In the original loop, there's no UD, and according to semantic,
    9160              :          init_expr should be 0 for lshr, ashl, and >>= (prec - 1) for ashr.  */
    9161           24 :       if (!early_exit_p
    9162           24 :           && wi::mul (wi::to_widest (step_expr),
    9163           36 :                       wi::to_widest (skip_niters)) >= prec)
    9164              :         {
    9165            6 :           if (induction_type == vect_step_op_shl
    9166            6 :               || TYPE_UNSIGNED (type))
    9167            4 :             init_expr = build_zero_cst (type);
    9168              :           else
    9169            2 :             init_expr = gimple_build (stmts, RSHIFT_EXPR, type,
    9170              :                                       init_expr,
    9171            4 :                                       wide_int_to_tree (type, prec - 1));
    9172              :         }
    9173              :       else
    9174              :         {
    9175              :           /* ???  For not constant skip_niters we'd have to treat the
    9176              :              overflow case.  */
    9177            6 :           gcc_assert (!early_exit_p);
    9178            6 :           step_expr = fold_build2 (MULT_EXPR, TREE_TYPE (skip_niters),
    9179              :                                    skip_niters,
    9180              :                                    fold_convert (TREE_TYPE (skip_niters),
    9181              :                                                  step_expr));
    9182            8 :           init_expr = fold_build2 ((induction_type == vect_step_op_shr
    9183              :                                     ? RSHIFT_EXPR : LSHIFT_EXPR),
    9184              :                                     type, init_expr, step_expr);
    9185            6 :           init_expr = force_gimple_operand (init_expr, stmts, false, NULL);
    9186              :         }
    9187              :       break;
    9188              : 
    9189           71 :     case vect_step_op_mul:
    9190           71 :       {
    9191              :         /* Due to UB we can't support vect_step_op_mul with early break for now.
    9192              :            so assert and block.  */
    9193           71 :         gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST);
    9194           71 :         tree utype = unsigned_type_for (type);
    9195           71 :         init_expr = gimple_convert (stmts, utype, init_expr);
    9196           71 :         wide_int skipn = wi::to_wide (skip_niters);
    9197           71 :         wide_int begin = wi::to_wide (step_expr);
    9198           71 :         auto_mpz base, exp, mod, res;
    9199           71 :         wi::to_mpz (begin, base, TYPE_SIGN (type));
    9200           71 :         wi::to_mpz (skipn, exp, UNSIGNED);
    9201           71 :         mpz_ui_pow_ui (mod, 2, TYPE_PRECISION (type));
    9202           71 :         mpz_powm (res, base, exp, mod);
    9203           71 :         begin = wi::from_mpz (utype, res, true);
    9204           71 :         tree mult_expr = wide_int_to_tree (utype, begin);
    9205           71 :         init_expr = gimple_build (stmts, MULT_EXPR, utype,
    9206              :                                   init_expr, mult_expr);
    9207           71 :         init_expr = gimple_convert (stmts, type, init_expr);
    9208           71 :       }
    9209           71 :       break;
    9210              : 
    9211            0 :     default:
    9212            0 :       gcc_unreachable ();
    9213              :     }
    9214              : 
    9215           83 :   return init_expr;
    9216              : }
    9217              : 
    9218              : /* Create vector step for vectorized iv.  */
    9219              : static tree
    9220         1221 : vect_create_nonlinear_iv_step (gimple_seq* stmts, tree step_expr,
    9221              :                                poly_uint64 vf,
    9222              :                                enum vect_induction_op_type induction_type)
    9223              : {
    9224         1221 :   tree expr = build_int_cst (TREE_TYPE (step_expr), vf);
    9225         1221 :   tree new_name = NULL;
    9226              :   /* Step should be pow (step, vf) for mult induction.  */
    9227         1221 :   if (induction_type == vect_step_op_mul)
    9228              :     {
    9229           78 :       gcc_assert (vf.is_constant ());
    9230           78 :       wide_int begin = wi::to_wide (step_expr);
    9231              : 
    9232          592 :       for (unsigned i = 0; i != vf.to_constant () - 1; i++)
    9233          514 :         begin = wi::mul (begin, wi::to_wide (step_expr));
    9234              : 
    9235           78 :       new_name = wide_int_to_tree (TREE_TYPE (step_expr), begin);
    9236           78 :     }
    9237         1143 :   else if (induction_type == vect_step_op_neg)
    9238              :     /* Do nothing.  */
    9239              :     ;
    9240              :   else
    9241           24 :     new_name = gimple_build (stmts, MULT_EXPR, TREE_TYPE (step_expr),
    9242              :                              expr, step_expr);
    9243         1221 :   return new_name;
    9244              : }
    9245              : 
    9246              : static tree
    9247         1221 : vect_create_nonlinear_iv_vec_step (loop_vec_info loop_vinfo,
    9248              :                                    stmt_vec_info stmt_info,
    9249              :                                    tree new_name, tree vectype,
    9250              :                                    enum vect_induction_op_type induction_type)
    9251              : {
    9252              :   /* No step is needed for neg induction.  */
    9253         1221 :   if (induction_type == vect_step_op_neg)
    9254              :     return NULL;
    9255              : 
    9256          102 :   tree t = unshare_expr (new_name);
    9257          102 :   gcc_assert (CONSTANT_CLASS_P (new_name)
    9258              :               || TREE_CODE (new_name) == SSA_NAME);
    9259          102 :   tree new_vec = build_vector_from_val (vectype, t);
    9260          102 :   tree vec_step = vect_init_vector (loop_vinfo, stmt_info,
    9261              :                                     new_vec, vectype, NULL);
    9262          102 :   return vec_step;
    9263              : }
    9264              : 
    9265              : /* Update vectorized iv with vect_step, induc_def is init.  */
    9266              : static tree
    9267         1427 : vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
    9268              :                           tree induc_def, tree vec_step,
    9269              :                           enum vect_induction_op_type induction_type)
    9270              : {
    9271         1427 :   tree vec_def = induc_def;
    9272         1427 :   switch (induction_type)
    9273              :     {
    9274           78 :     case vect_step_op_mul:
    9275           78 :       {
    9276              :         /* Use unsigned mult to avoid UD integer overflow.  */
    9277           78 :         tree uvectype = unsigned_type_for (vectype);
    9278           78 :         vec_def = gimple_convert (stmts, uvectype, vec_def);
    9279           78 :         vec_step = gimple_convert (stmts, uvectype, vec_step);
    9280           78 :         vec_def = gimple_build (stmts, MULT_EXPR, uvectype,
    9281              :                                 vec_def, vec_step);
    9282           78 :         vec_def = gimple_convert (stmts, vectype, vec_def);
    9283              :       }
    9284           78 :       break;
    9285              : 
    9286           36 :     case vect_step_op_shr:
    9287           36 :       vec_def = gimple_build (stmts, RSHIFT_EXPR, vectype,
    9288              :                               vec_def, vec_step);
    9289           36 :       break;
    9290              : 
    9291            6 :     case vect_step_op_shl:
    9292            6 :       vec_def = gimple_build (stmts, LSHIFT_EXPR, vectype,
    9293              :                               vec_def, vec_step);
    9294            6 :       break;
    9295              :     case vect_step_op_neg:
    9296              :       vec_def = induc_def;
    9297              :       /* Do nothing.  */
    9298              :       break;
    9299            0 :     default:
    9300            0 :       gcc_unreachable ();
    9301              :     }
    9302              : 
    9303         1427 :   return vec_def;
    9304              : 
    9305              : }
    9306              : 
    9307              : /* Function vectorizable_nonlinear_induction
    9308              : 
    9309              :    Check if STMT_INFO performs an nonlinear induction computation that can be
    9310              :    vectorized. If VEC_STMT is also passed, vectorize the induction PHI: create
    9311              :    a vectorized phi to replace it, put it in VEC_STMT, and add it to the same
    9312              :    basic block.
    9313              :    Return true if STMT_INFO is vectorizable in this way.  */
    9314              : 
    9315              : static bool
    9316        13483 : vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
    9317              :                                   stmt_vec_info stmt_info,
    9318              :                                   slp_tree slp_node,
    9319              :                                   stmt_vector_for_cost *cost_vec)
    9320              : {
    9321        13483 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    9322        13483 :   unsigned ncopies;
    9323        13483 :   bool nested_in_vect_loop = false;
    9324        13483 :   class loop *iv_loop;
    9325        13483 :   tree vec_def;
    9326        13483 :   edge pe = loop_preheader_edge (loop);
    9327        13483 :   basic_block new_bb;
    9328        13483 :   tree vec_init, vec_step;
    9329        13483 :   tree new_name;
    9330        13483 :   gimple *new_stmt;
    9331        13483 :   gphi *induction_phi;
    9332        13483 :   tree induc_def, vec_dest;
    9333        13483 :   tree init_expr, step_expr;
    9334        13483 :   tree niters_skip;
    9335        13483 :   poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    9336        13483 :   unsigned i;
    9337        13483 :   gimple_stmt_iterator si;
    9338              : 
    9339        13483 :   gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
    9340              : 
    9341        13483 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    9342        13483 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    9343        13483 :   enum vect_induction_op_type induction_type
    9344              :     = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
    9345              : 
    9346        13483 :   gcc_assert (induction_type > vect_step_op_add);
    9347              : 
    9348        13483 :   ncopies = vect_get_num_copies (loop_vinfo, slp_node);
    9349        13483 :   gcc_assert (ncopies >= 1);
    9350              : 
    9351              :   /* FORNOW. Only handle nonlinear induction in the same loop.  */
    9352        13483 :   if (nested_in_vect_loop_p (loop, stmt_info))
    9353              :     {
    9354            0 :       if (dump_enabled_p ())
    9355            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9356              :                          "nonlinear induction in nested loop.\n");
    9357              :       return false;
    9358              :     }
    9359              : 
    9360        13483 :   iv_loop = loop;
    9361        13483 :   gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
    9362              : 
    9363              :   /* TODO: Support multi-lane SLP for nonlinear iv. There should be separate
    9364              :      vector iv update for each iv and a permutation to generate wanted
    9365              :      vector iv.  */
    9366        13483 :   if (SLP_TREE_LANES (slp_node) > 1)
    9367              :     {
    9368            0 :       if (dump_enabled_p ())
    9369            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9370              :                          "SLP induction not supported for nonlinear"
    9371              :                          " induction.\n");
    9372              :       return false;
    9373              :     }
    9374              : 
    9375        13483 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
    9376              :     {
    9377            0 :       if (dump_enabled_p ())
    9378            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9379              :                          "floating point nonlinear induction vectorization"
    9380              :                          " not supported.\n");
    9381              :       return false;
    9382              :     }
    9383              : 
    9384        13483 :   step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
    9385        13483 :   init_expr = vect_phi_initial_value (phi);
    9386        13483 :   gcc_assert (step_expr != NULL_TREE && init_expr != NULL
    9387              :               && TREE_CODE (step_expr) == INTEGER_CST);
    9388              :   /* step_expr should be aligned with init_expr,
    9389              :      .i.e. uint64 a >> 1, step is int, but vector<uint64> shift is used.  */
    9390        13483 :   step_expr = fold_convert (TREE_TYPE (vectype), step_expr);
    9391              : 
    9392        13483 :   if (TREE_CODE (init_expr) == INTEGER_CST)
    9393         4141 :     init_expr = fold_convert (TREE_TYPE (vectype), init_expr);
    9394         9342 :   else if (!tree_nop_conversion_p (TREE_TYPE (vectype), TREE_TYPE (init_expr)))
    9395              :     {
    9396              :       /* INIT_EXPR could be a bit_field, bail out for such case.  */
    9397            4 :       if (dump_enabled_p ())
    9398            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9399              :                          "nonlinear induction vectorization failed:"
    9400              :                          " component type of vectype is not a nop conversion"
    9401              :                          " from type of init_expr.\n");
    9402              :       return false;
    9403              :     }
    9404              : 
    9405        13479 :   switch (induction_type)
    9406              :     {
    9407         3719 :     case vect_step_op_neg:
    9408         3719 :       if (maybe_eq (TYPE_VECTOR_SUBPARTS (vectype), 1u))
    9409              :         return false;
    9410         3557 :       if (TREE_CODE (init_expr) != INTEGER_CST
    9411          285 :           && TREE_CODE (init_expr) != REAL_CST)
    9412              :         {
    9413              :           /* Check for backend support of NEGATE_EXPR and vec_perm.  */
    9414          285 :           if (!directly_supported_p (NEGATE_EXPR, vectype))
    9415            0 :             return false;
    9416              : 
    9417              :           /* The encoding has 2 interleaved stepped patterns.  */
    9418          285 :           vec_perm_builder sel (nunits, 2, 3);
    9419          285 :           machine_mode mode = TYPE_MODE (vectype);
    9420          285 :           sel.quick_grow (6);
    9421         1425 :           for (i = 0; i < 3; i++)
    9422              :             {
    9423          855 :               sel[i * 2] = i;
    9424          855 :               sel[i * 2 + 1] = i + nunits;
    9425              :             }
    9426          285 :           vec_perm_indices indices (sel, 2, nunits);
    9427          285 :           if (!can_vec_perm_const_p (mode, mode, indices))
    9428            0 :             return false;
    9429          285 :         }
    9430              :       break;
    9431              : 
    9432         1072 :     case vect_step_op_mul:
    9433         1072 :       {
    9434              :         /* Check for backend support of MULT_EXPR.  */
    9435         1072 :         if (!directly_supported_p (MULT_EXPR, vectype))
    9436              :           return false;
    9437              : 
    9438              :         /* ?? How to construct vector step for variable number vector.
    9439              :            [ 1, step, pow (step, 2), pow (step, 4), .. ].  */
    9440              :         if (!vf.is_constant ())
    9441              :           return false;
    9442              :       }
    9443              :       break;
    9444              : 
    9445         8286 :     case vect_step_op_shr:
    9446              :       /* Check for backend support of RSHIFT_EXPR.  */
    9447         8286 :       if (!directly_supported_p (RSHIFT_EXPR, vectype, optab_vector))
    9448              :         return false;
    9449              : 
    9450              :       /* Don't shift more than type precision to avoid UB.  */
    9451           98 :       if (!tree_fits_uhwi_p (step_expr)
    9452           98 :           || maybe_ge (vf * tree_to_uhwi (step_expr),
    9453              :                        TYPE_PRECISION (TREE_TYPE (init_expr))))
    9454              :         return false;
    9455              :       break;
    9456              : 
    9457          402 :     case vect_step_op_shl:
    9458              :       /* Check for backend support of RSHIFT_EXPR.  */
    9459          402 :       if (!directly_supported_p (LSHIFT_EXPR, vectype, optab_vector))
    9460              :         return false;
    9461              : 
    9462              :       /* Don't shift more than type precision to avoid UB.  */
    9463           42 :       if (!tree_fits_uhwi_p (step_expr)
    9464           42 :           || maybe_ge (vf * tree_to_uhwi (step_expr),
    9465              :                        TYPE_PRECISION (TREE_TYPE (init_expr))))
    9466              :         return false;
    9467              :       break;
    9468              : 
    9469            0 :     default:
    9470            0 :       gcc_unreachable ();
    9471              :     }
    9472              : 
    9473         4491 :   if (cost_vec) /* transformation not required.  */
    9474              :     {
    9475         3559 :       unsigned inside_cost = 0, prologue_cost = 0;
    9476              :       /* loop cost for vec_loop. Neg induction doesn't have any
    9477              :          inside_cost.  */
    9478         3559 :       inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
    9479              :                                       slp_node, 0, vect_body);
    9480              : 
    9481              :       /* loop cost for vec_loop. Neg induction doesn't have any
    9482              :          inside_cost.  */
    9483         3559 :       if (induction_type == vect_step_op_neg)
    9484         2724 :         inside_cost = 0;
    9485              : 
    9486              :       /* prologue cost for vec_init and vec_step.  */
    9487         3559 :       prologue_cost = record_stmt_cost (cost_vec, 2, scalar_to_vec,
    9488              :                                         slp_node, 0, vect_prologue);
    9489              : 
    9490         3559 :       if (dump_enabled_p ())
    9491          133 :         dump_printf_loc (MSG_NOTE, vect_location,
    9492              :                          "vect_model_induction_cost: inside_cost = %d, "
    9493              :                          "prologue_cost = %d. \n", inside_cost,
    9494              :                          prologue_cost);
    9495              : 
    9496         3559 :       SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
    9497         3559 :       DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
    9498         3559 :       return true;
    9499              :     }
    9500              : 
    9501              :   /* Transform.  */
    9502              : 
    9503              :   /* Compute a vector variable, initialized with the first VF values of
    9504              :      the induction variable.  E.g., for an iv with IV_PHI='X' and
    9505              :      evolution S, for a vector of 4 units, we want to compute:
    9506              :      [X, X + S, X + 2*S, X + 3*S].  */
    9507              : 
    9508          932 :   if (dump_enabled_p ())
    9509           43 :     dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
    9510              : 
    9511          932 :   pe = loop_preheader_edge (iv_loop);
    9512              :   /* Find the first insertion point in the BB.  */
    9513          932 :   basic_block bb = gimple_bb (phi);
    9514          932 :   si = gsi_after_labels (bb);
    9515              : 
    9516          932 :   gimple_seq stmts = NULL;
    9517              : 
    9518          932 :   niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
    9519              :   /* If we are using the loop mask to "peel" for alignment then we need
    9520              :      to adjust the start value here.  */
    9521          932 :   if (niters_skip != NULL_TREE)
    9522            0 :     init_expr = vect_peel_nonlinear_iv_init (&stmts, init_expr, niters_skip,
    9523              :                                              step_expr, induction_type, false);
    9524              : 
    9525          932 :   vec_init = vect_create_nonlinear_iv_init (&stmts, init_expr,
    9526              :                                             step_expr, nunits, vectype,
    9527              :                                             induction_type);
    9528          932 :   if (stmts)
    9529              :     {
    9530          167 :       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
    9531          167 :       gcc_assert (!new_bb);
    9532              :     }
    9533              : 
    9534          932 :   stmts = NULL;
    9535          932 :   new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
    9536              :                                             vf, induction_type);
    9537          932 :   if (stmts)
    9538              :     {
    9539            0 :       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
    9540            0 :       gcc_assert (!new_bb);
    9541              :     }
    9542              : 
    9543          932 :   vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
    9544              :                                                 new_name, vectype,
    9545              :                                                 induction_type);
    9546              :   /* Create the following def-use cycle:
    9547              :      loop prolog:
    9548              :      vec_init = ...
    9549              :      vec_step = ...
    9550              :      loop:
    9551              :      vec_iv = PHI <vec_init, vec_loop>
    9552              :      ...
    9553              :      STMT
    9554              :      ...
    9555              :      vec_loop = vec_iv + vec_step;  */
    9556              : 
    9557              :   /* Create the induction-phi that defines the induction-operand.  */
    9558          932 :   vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
    9559          932 :   induction_phi = create_phi_node (vec_dest, iv_loop->header);
    9560          932 :   induc_def = PHI_RESULT (induction_phi);
    9561              : 
    9562              :   /* Create the iv update inside the loop.  */
    9563          932 :   stmts = NULL;
    9564          932 :   vec_def = vect_update_nonlinear_iv (&stmts, vectype,
    9565              :                                       induc_def, vec_step,
    9566              :                                       induction_type);
    9567              : 
    9568          932 :   gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
    9569          932 :   new_stmt = SSA_NAME_DEF_STMT (vec_def);
    9570              : 
    9571              :   /* Set the arguments of the phi node:  */
    9572          932 :   add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
    9573          932 :   add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
    9574              :                UNKNOWN_LOCATION);
    9575              : 
    9576          932 :   slp_node->push_vec_def (induction_phi);
    9577              : 
    9578              :   /* In case that vectorization factor (VF) is bigger than the number
    9579              :      of elements that we can fit in a vectype (nunits), we have to generate
    9580              :      more than one vector stmt - i.e - we need to "unroll" the
    9581              :      vector stmt by a factor VF/nunits.  For more details see documentation
    9582              :      in vectorizable_operation.  */
    9583              : 
    9584          932 :   if (ncopies > 1)
    9585              :     {
    9586          289 :       stmts = NULL;
    9587              :       /* FORNOW. This restriction should be relaxed.  */
    9588          289 :       gcc_assert (!nested_in_vect_loop);
    9589              : 
    9590          289 :       new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
    9591              :                                                 nunits, induction_type);
    9592              : 
    9593          289 :       vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
    9594              :                                                     new_name, vectype,
    9595              :                                                     induction_type);
    9596          289 :       vec_def = induc_def;
    9597         1073 :       for (i = 1; i < ncopies; i++)
    9598              :         {
    9599              :           /* vec_i = vec_prev + vec_step.  */
    9600          495 :           stmts = NULL;
    9601          495 :           vec_def = vect_update_nonlinear_iv (&stmts, vectype,
    9602              :                                               vec_def, vec_step,
    9603              :                                               induction_type);
    9604          495 :           gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
    9605          495 :           new_stmt = SSA_NAME_DEF_STMT (vec_def);
    9606          495 :           slp_node->push_vec_def (new_stmt);
    9607              :         }
    9608              :     }
    9609              : 
    9610          932 :   if (dump_enabled_p ())
    9611           86 :     dump_printf_loc (MSG_NOTE, vect_location,
    9612              :                      "transform induction: created def-use cycle: %G%G",
    9613           43 :                      (gimple *) induction_phi, SSA_NAME_DEF_STMT (vec_def));
    9614              : 
    9615              :   return true;
    9616              : }
    9617              : 
    9618              : /* Return true if the scalar initial values and steps of the SLP induction
    9619              :    lanes allow the first CANDIDATE_NIVS IVs to be reused circularly for the
    9620              :    remaining lanes.  */
    9621              : static bool
    9622           26 : vect_slp_induction_reuse_p (tree *steps, tree *inits, unsigned group_size,
    9623              :                           unsigned HOST_WIDE_INT const_nunits,
    9624              :                           unsigned candidate_nivs, unsigned nivs)
    9625              : {
    9626           26 :   gcc_assert (candidate_nivs > 0);
    9627           26 :   gcc_assert (candidate_nivs < nivs);
    9628              : 
    9629              :   /* This function compares only STEPS and INITS, so all checked lanes must
    9630              :      precede the first wrap of the SLP group.  */
    9631           26 :   gcc_assert (nivs * const_nunits <= group_size);
    9632              : 
    9633           62 :   for (unsigned ivn = candidate_nivs; ivn < nivs; ++ivn)
    9634              :     {
    9635           42 :       unsigned reuse_ivn = ivn % candidate_nivs;
    9636          172 :       for (unsigned HOST_WIDE_INT eltn = 0; eltn < const_nunits; ++eltn)
    9637              :         {
    9638          136 :           unsigned HOST_WIDE_INT elt = ivn * const_nunits + eltn;
    9639          136 :           unsigned HOST_WIDE_INT reused_elt
    9640          136 :             = reuse_ivn * const_nunits + eltn;
    9641              : 
    9642          136 :           if (!operand_equal_p (steps[elt], steps[reused_elt], 0)
    9643          136 :               || !operand_equal_p (inits[elt], inits[reused_elt], 0))
    9644              :             return false;
    9645              :         }
    9646              :     }
    9647              : 
    9648              :   return true;
    9649              : }
    9650              : 
    9651              : /* Function vectorizable_induction
    9652              : 
    9653              :    Check if STMT_INFO performs an induction computation that can be vectorized.
    9654              :    If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
    9655              :    phi to replace it, put it in VEC_STMT, and add it to the same basic block.
    9656              :    Return true if STMT_INFO is vectorizable in this way.  */
    9657              : 
    9658              : bool
    9659       351732 : vectorizable_induction (loop_vec_info loop_vinfo,
    9660              :                         stmt_vec_info stmt_info,
    9661              :                         slp_tree slp_node, stmt_vector_for_cost *cost_vec)
    9662              : {
    9663       351732 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    9664       351732 :   bool nested_in_vect_loop = false;
    9665       351732 :   class loop *iv_loop;
    9666       351732 :   tree vec_def;
    9667       351732 :   edge pe = loop_preheader_edge (loop);
    9668       351732 :   basic_block new_bb;
    9669       351732 :   tree vec_init = NULL_TREE, vec_step, t;
    9670       351732 :   tree new_name;
    9671       351732 :   gphi *induction_phi;
    9672       351732 :   tree induc_def, vec_dest;
    9673       351732 :   unsigned i;
    9674       351732 :   tree index_vectype = NULL_TREE;
    9675       351732 :   gimple_stmt_iterator si;
    9676       351732 :   enum vect_induction_op_type induction_type
    9677              :     = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
    9678              : 
    9679       351732 :   gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
    9680       189615 :   if (!phi)
    9681              :     return false;
    9682              : 
    9683       189615 :   if (!STMT_VINFO_RELEVANT_P (stmt_info))
    9684              :     return false;
    9685              : 
    9686              :   /* Make sure it was recognized as induction computation.  */
    9687       189615 :   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
    9688              :     return false;
    9689              : 
    9690       746823 :   for (auto phi_info : SLP_TREE_SCALAR_STMTS (slp_node))
    9691       190395 :     if (phi_info)
    9692              :       {
    9693       190395 :         gcc_assert (STMT_VINFO_DEF_TYPE (phi_info) == vect_induction_def);
    9694       190395 :         if (STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (phi_info) != induction_type)
    9695              :           return false;
    9696              :       }
    9697              : 
    9698              :   /* Handle nonlinear induction in a separate place.  */
    9699       185474 :   if (induction_type != vect_step_op_add)
    9700        13483 :     return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
    9701        13483 :                                              slp_node, cost_vec);
    9702              : 
    9703       171991 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
    9704       171991 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
    9705              : 
    9706              :   /* FORNOW. These restrictions should be relaxed.  */
    9707       171991 :   if (nested_in_vect_loop_p (loop, stmt_info))
    9708              :     {
    9709          780 :       imm_use_iterator imm_iter;
    9710          780 :       use_operand_p use_p;
    9711          780 :       gimple *exit_phi;
    9712          780 :       edge latch_e;
    9713          780 :       tree loop_arg;
    9714              : 
    9715          780 :       exit_phi = NULL;
    9716          780 :       latch_e = loop_latch_edge (loop->inner);
    9717          780 :       loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
    9718         1597 :       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
    9719              :         {
    9720          843 :           gimple *use_stmt = USE_STMT (use_p);
    9721          843 :           if (is_gimple_debug (use_stmt))
    9722           36 :             continue;
    9723              : 
    9724          807 :           if (!flow_bb_inside_loop_p (loop->inner, gimple_bb (use_stmt)))
    9725              :             {
    9726              :               exit_phi = use_stmt;
    9727              :               break;
    9728              :             }
    9729          780 :         }
    9730          780 :       if (exit_phi)
    9731              :         {
    9732           26 :           stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi);
    9733           26 :           if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
    9734           10 :                 && !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
    9735              :             {
    9736           16 :               if (dump_enabled_p ())
    9737           16 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9738              :                                  "inner-loop induction only used outside "
    9739              :                                  "of the outer vectorized loop.\n");
    9740           16 :               return false;
    9741              :             }
    9742              :         }
    9743              : 
    9744          764 :       nested_in_vect_loop = true;
    9745          764 :       iv_loop = loop->inner;
    9746              :     }
    9747              :   else
    9748              :     iv_loop = loop;
    9749       171975 :   gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
    9750              : 
    9751       171975 :   if (!nunits.is_constant () && SLP_TREE_LANES (slp_node) != 1)
    9752              :     {
    9753              :       /* The current SLP code creates the step value element-by-element.  */
    9754              :       if (dump_enabled_p ())
    9755              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9756              :                          "SLP induction not supported for variable-length"
    9757              :                          " vectors.\n");
    9758              :       return false;
    9759              :     }
    9760              : 
    9761       171975 :   if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
    9762              :     {
    9763           12 :       if (dump_enabled_p ())
    9764           12 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9765              :                          "floating point induction vectorization disabled\n");
    9766              :       return false;
    9767              :     }
    9768              : 
    9769       171963 :   tree step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
    9770       171963 :   gcc_assert (step_expr != NULL_TREE);
    9771       343902 :   if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
    9772       343801 :       && !type_has_mode_precision_p (TREE_TYPE (step_expr)))
    9773              :     {
    9774           12 :       if (dump_enabled_p ())
    9775           12 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9776              :                          "bit-precision induction vectorization not "
    9777              :                          "supported.\n");
    9778              :       return false;
    9779              :     }
    9780       171951 :   tree stept = TREE_TYPE (step_expr);
    9781       171951 :   tree step_vectype
    9782       171951 :     = (!INTEGRAL_TYPE_P (stept) ? vectype
    9783       171850 :        : signed_or_unsigned_type_for (TYPE_UNSIGNED (stept), vectype));
    9784       171951 :   stept = TREE_TYPE (step_vectype);
    9785              : 
    9786              :   /* Check for target support of the vectorized arithmetic used here.  */
    9787       171951 :   if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
    9788       171951 :       || !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default))
    9789              :       return false;
    9790       140251 :   if (!nunits.is_constant ()
    9791       140251 :       || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
    9792              :     {
    9793            0 :       if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default))
    9794              :         return false;
    9795              :       /* FLOAT_EXPR when computing VEC_INIT for float inductions.  */
    9796            0 :       if (SCALAR_FLOAT_TYPE_P (stept))
    9797              :         {
    9798            0 :           tree index_type = build_nonstandard_integer_type
    9799            0 :                 (GET_MODE_BITSIZE (SCALAR_TYPE_MODE (stept)), 1);
    9800              : 
    9801            0 :           index_vectype = build_vector_type (index_type, nunits);
    9802            0 :           if (!can_float_p (TYPE_MODE (step_vectype),
    9803            0 :                             TYPE_MODE (index_vectype), 1))
    9804              :             return false;
    9805              :         }
    9806              :     }
    9807              : 
    9808       140251 :   unsigned nvects = vect_get_num_copies (loop_vinfo, slp_node);
    9809       140251 :   if (cost_vec) /* transformation not required.  */
    9810              :     {
    9811       372516 :       unsigned inside_cost = 0, prologue_cost = 0;
    9812              :       /* We eventually need to set a vector type on invariant
    9813              :          arguments.  */
    9814              :       unsigned j;
    9815              :       slp_tree child;
    9816       372516 :       FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
    9817       248344 :         if (!vect_maybe_update_slp_op_vectype
    9818       248344 :             (child, SLP_TREE_VECTYPE (slp_node)))
    9819              :           {
    9820            0 :             if (dump_enabled_p ())
    9821            0 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    9822              :                                "incompatible vector types for "
    9823              :                                "invariants\n");
    9824              :             return false;
    9825              :           }
    9826              :       /* loop cost for vec_loop.  */
    9827       124172 :       inside_cost = record_stmt_cost (cost_vec, nvects,
    9828              :                                       vector_stmt, slp_node, 0, vect_body);
    9829              :       /* prologue cost for vec_init (if not nested) and step.  */
    9830       124172 :       prologue_cost = record_stmt_cost (cost_vec, 1 + !nested_in_vect_loop,
    9831              :                                         scalar_to_vec,
    9832              :                                         slp_node, 0, vect_prologue);
    9833       124172 :       if (dump_enabled_p ())
    9834         4373 :         dump_printf_loc (MSG_NOTE, vect_location,
    9835              :                          "vect_model_induction_cost: inside_cost = %d, "
    9836              :                          "prologue_cost = %d .\n", inside_cost,
    9837              :                          prologue_cost);
    9838              : 
    9839       124172 :       SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
    9840       124172 :       DUMP_VECT_SCOPE ("vectorizable_induction");
    9841       124172 :       return true;
    9842              :     }
    9843              : 
    9844              :   /* Transform.  */
    9845              : 
    9846              :   /* Compute a vector variable, initialized with the first VF values of
    9847              :      the induction variable.  E.g., for an iv with IV_PHI='X' and
    9848              :      evolution S, for a vector of 4 units, we want to compute:
    9849              :      [X, X + S, X + 2*S, X + 3*S].  */
    9850              : 
    9851        16079 :   if (dump_enabled_p ())
    9852         2926 :     dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
    9853              : 
    9854        16079 :   pe = loop_preheader_edge (iv_loop);
    9855              :   /* Find the first insertion point in the BB.  */
    9856        16079 :   basic_block bb = gimple_bb (phi);
    9857        16079 :   si = gsi_after_labels (bb);
    9858              : 
    9859              :   /* For SLP induction we have to generate several IVs as for example
    9860              :      with group size 3 we need
    9861              :        [i0, i1, i2, i0 + S0] [i1 + S1, i2 + S2, i0 + 2*S0, i1 + 2*S1]
    9862              :        [i2 + 2*S2, i0 + 3*S0, i1 + 3*S1, i2 + 3*S2].  */
    9863        16079 :   gimple_stmt_iterator incr_si;
    9864        16079 :   bool insert_after;
    9865        16079 :   standard_iv_increment_position (iv_loop, &incr_si, &insert_after);
    9866              : 
    9867              :   /* The initial values are vectorized, but any lanes > group_size
    9868              :      need adjustment.  */
    9869        16079 :   slp_tree init_node
    9870        16079 :       = SLP_TREE_CHILDREN (slp_node)[pe->dest_idx];
    9871              : 
    9872              :   /* Gather steps.  Since we do not vectorize inductions as
    9873              :      cycles we have to reconstruct the step from SCEV data.  */
    9874        16079 :   unsigned group_size = SLP_TREE_LANES (slp_node);
    9875        16079 :   tree *steps = XALLOCAVEC (tree, group_size);
    9876        16079 :   tree *inits = XALLOCAVEC (tree, group_size);
    9877        16079 :   stmt_vec_info phi_info;
    9878        49663 :   FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, phi_info)
    9879              :     {
    9880        17505 :       steps[i] = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
    9881        17505 :       if (!init_node)
    9882        17277 :         inits[i] = gimple_phi_arg_def (as_a<gphi *> (phi_info->stmt),
    9883              :                                        pe->dest_idx);
    9884              :     }
    9885              : 
    9886              :   /* Now generate the IVs.  */
    9887        32158 :   gcc_assert (multiple_p (nunits * nvects, group_size));
    9888        16079 :   unsigned nivs;
    9889        16079 :   unsigned HOST_WIDE_INT const_nunits;
    9890        16079 :   if (nested_in_vect_loop)
    9891              :     nivs = nvects;
    9892        15872 :   else if (nunits.is_constant (&const_nunits)
    9893        15872 :            && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
    9894              :     {
    9895        15872 :       gcc_assert (!init_node);
    9896              :       /* Compute the number of distinct IVs we need.  We can reduce the
    9897              :          number when later vector chunks are equal to earlier chunks.  */
    9898        15872 :       nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
    9899        15872 :       unsigned group_sizep = group_size;
    9900        15872 :       if (group_sizep % const_nunits == 0)
    9901              :         {
    9902          124 :           group_sizep = group_sizep / const_nunits;
    9903          124 :           unsigned candidate_nivs
    9904          124 :             = least_common_multiple (group_sizep, const_nunits) / const_nunits;
    9905          124 :           if (candidate_nivs < nivs
    9906          124 :               && vect_slp_induction_reuse_p (steps, inits, group_size,
    9907              :                                            const_nunits, candidate_nivs, nivs))
    9908              :             {
    9909           20 :               if (dump_enabled_p ())
    9910           16 :                 dump_printf_loc (MSG_NOTE, vect_location,
    9911              :                                  "reusing %u SLP induction IVs for %u "
    9912              :                                  "vector chunks\n",
    9913              :                                  candidate_nivs, nivs);
    9914              :               nivs = candidate_nivs;
    9915              :             }
    9916              :         }
    9917              :     }
    9918              :   else
    9919              :     {
    9920            0 :       gcc_assert (SLP_TREE_LANES (slp_node) == 1);
    9921              :       nivs = 1;
    9922              :     }
    9923        16079 :   gimple_seq init_stmts = NULL;
    9924        16079 :   gimple_seq lupdate_mul_stmts = NULL;
    9925        16079 :   tree lupdate_mul = NULL_TREE;
    9926        16079 :   if (!nested_in_vect_loop)
    9927              :     {
    9928        15872 :       if (nunits.is_constant (&const_nunits)
    9929        15872 :           && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
    9930              :         {
    9931              :           /* The number of iterations covered in one vector iteration.  */
    9932        15872 :           unsigned lup_mul = (nvects * const_nunits) / group_size;
    9933        15872 :           lupdate_mul
    9934        15872 :             = build_vector_from_val (step_vectype,
    9935        15872 :                                      SCALAR_FLOAT_TYPE_P (stept)
    9936           28 :                                      ? build_real_from_wide (stept, lup_mul,
    9937              :                                                              UNSIGNED)
    9938        31716 :                                      : build_int_cstu (stept, lup_mul));
    9939              :         }
    9940              :       else
    9941              :         {
    9942            0 :           gimple_seq *update_stmts
    9943              :             = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
    9944              :                 ? &init_stmts
    9945              :                 : &lupdate_mul_stmts;
    9946            0 :           if (SCALAR_FLOAT_TYPE_P (stept))
    9947              :             {
    9948            0 :               tree increment
    9949            0 :                 = gimple_convert (update_stmts, integer_type_node,
    9950              :                                   LOOP_VINFO_IV_INCREMENT (loop_vinfo));
    9951            0 :               lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept,
    9952              :                                           increment);
    9953              :             }
    9954              :           else
    9955            0 :             lupdate_mul = gimple_convert (update_stmts, stept,
    9956              :                                          LOOP_VINFO_IV_INCREMENT (loop_vinfo));
    9957            0 :           lupdate_mul = gimple_build_vector_from_val (update_stmts,
    9958              :                                                       step_vectype,
    9959              :                                                       lupdate_mul);
    9960              :         }
    9961              :     }
    9962        16079 :   tree peel_mul = NULL_TREE;
    9963        16079 :   if (LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo))
    9964              :     {
    9965            0 :       if (SCALAR_FLOAT_TYPE_P (stept))
    9966            0 :         peel_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept,
    9967              :                                  LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
    9968              :       else
    9969            0 :         peel_mul = gimple_convert (&init_stmts, stept,
    9970              :                                    LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
    9971            0 :       peel_mul = gimple_build_vector_from_val (&init_stmts,
    9972              :                                                step_vectype, peel_mul);
    9973              :     }
    9974        16079 :   tree step_mul = NULL_TREE;
    9975        16079 :   unsigned ivn;
    9976        16079 :   auto_vec<tree> vec_steps;
    9977        32746 :   for (ivn = 0; ivn < nivs; ++ivn)
    9978              :     {
    9979        16667 :       gimple_seq stmts = NULL;
    9980        16667 :       bool invariant = true;
    9981        16667 :       if (nunits.is_constant (&const_nunits)
    9982        16667 :           && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
    9983              :         {
    9984        16667 :           tree_vector_builder step_elts (step_vectype, const_nunits, 1);
    9985        16667 :           tree_vector_builder init_elts (vectype, const_nunits, 1);
    9986        16667 :           tree_vector_builder mul_elts (step_vectype, const_nunits, 1);
    9987       122954 :           for (unsigned eltn = 0; eltn < const_nunits; ++eltn)
    9988              :             {
    9989              :               /* The scalar steps of the IVs.  */
    9990        89620 :               tree elt = steps[(ivn*const_nunits + eltn) % group_size];
    9991        89620 :               elt = gimple_convert (&init_stmts, TREE_TYPE (step_vectype), elt);
    9992        89620 :               step_elts.quick_push (elt);
    9993        89620 :               if (!init_node)
    9994              :                 {
    9995              :                   /* The scalar inits of the IVs if not vectorized.  */
    9996        88414 :                   elt = inits[(ivn*const_nunits + eltn) % group_size];
    9997        88414 :                   if (!useless_type_conversion_p (TREE_TYPE (vectype),
    9998        88414 :                                                   TREE_TYPE (elt)))
    9999          324 :                     elt = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
   10000          324 :                                         TREE_TYPE (vectype), elt);
   10001        88414 :                   init_elts.quick_push (elt);
   10002              :                 }
   10003              :               /* The number of steps to add to the initial values.  */
   10004        89620 :               unsigned mul_elt = (ivn*const_nunits + eltn) / group_size;
   10005       179240 :               mul_elts.quick_push (SCALAR_FLOAT_TYPE_P (stept)
   10006       179138 :                                    ? build_real_from_wide (stept, mul_elt,
   10007              :                                                            UNSIGNED)
   10008       179138 :                                    : build_int_cstu (stept, mul_elt));
   10009              :             }
   10010        16667 :           vec_step = gimple_build_vector (&init_stmts, &step_elts);
   10011        16667 :           step_mul = gimple_build_vector (&init_stmts, &mul_elts);
   10012        16667 :           if (!init_node)
   10013        16426 :             vec_init = gimple_build_vector (&init_stmts, &init_elts);
   10014        16667 :         }
   10015              :       else
   10016              :         {
   10017            0 :           tree step = gimple_convert (&init_stmts, stept, steps[0]);
   10018            0 :           if (init_node)
   10019              :             ;
   10020            0 :           else if (INTEGRAL_TYPE_P (stept))
   10021              :             {
   10022            0 :               new_name = gimple_convert (&init_stmts, stept, inits[0]);
   10023              :               /* Build the initial value directly as a VEC_SERIES_EXPR.  */
   10024            0 :               vec_init = gimple_build (&init_stmts, VEC_SERIES_EXPR,
   10025              :                                        step_vectype, new_name, step);
   10026            0 :               if (!useless_type_conversion_p (vectype, step_vectype))
   10027            0 :                 vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
   10028              :                                          vectype, vec_init);
   10029              :             }
   10030              :           else
   10031              :             {
   10032              :               /* Build:
   10033              :                  [base, base, base, ...]
   10034              :                  + (vectype) [0, 1, 2, ...] * [step, step, step, ...].  */
   10035            0 :               gcc_assert (SCALAR_FLOAT_TYPE_P (stept));
   10036            0 :               gcc_assert (flag_associative_math);
   10037            0 :               gcc_assert (index_vectype != NULL_TREE);
   10038              : 
   10039            0 :               tree index = build_index_vector (index_vectype, 0, 1);
   10040            0 :               new_name = gimple_convert (&init_stmts, stept, inits[0]);
   10041            0 :               tree base_vec = gimple_build_vector_from_val (&init_stmts,
   10042              :                                                             step_vectype,
   10043              :                                                             new_name);
   10044            0 :               tree step_vec = gimple_build_vector_from_val (&init_stmts,
   10045              :                                                             step_vectype,
   10046              :                                                             step);
   10047            0 :               vec_init = gimple_build (&init_stmts, FLOAT_EXPR,
   10048              :                                        step_vectype, index);
   10049            0 :               vec_init = gimple_build (&init_stmts, MULT_EXPR,
   10050              :                                        step_vectype, vec_init, step_vec);
   10051            0 :               vec_init = gimple_build (&init_stmts, PLUS_EXPR,
   10052              :                                        step_vectype, vec_init, base_vec);
   10053            0 :               if (!useless_type_conversion_p (vectype, step_vectype))
   10054            0 :                 vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
   10055              :                                          vectype, vec_init);
   10056              :             }
   10057              :           /* iv_loop is nested in the loop to be vectorized. Generate:
   10058              :              vec_step = [S, S, S, S]  */
   10059            0 :           t = unshare_expr (step);
   10060            0 :           gcc_assert (CONSTANT_CLASS_P (t)
   10061              :                       || TREE_CODE (t) == SSA_NAME);
   10062            0 :           vec_step = gimple_build_vector_from_val (&init_stmts,
   10063              :                                                    step_vectype, t);
   10064              :         }
   10065        16667 :       vec_steps.safe_push (vec_step);
   10066        16667 :       if (peel_mul)
   10067              :         {
   10068            0 :           if (!step_mul)
   10069              :             {
   10070            0 :               gcc_assert (!nunits.is_constant ());
   10071              :               step_mul = gimple_build (&init_stmts,
   10072              :                                        MINUS_EXPR, step_vectype,
   10073              :                                        build_zero_cst (step_vectype), peel_mul);
   10074              :             }
   10075              :           else
   10076            0 :             step_mul = gimple_build (&init_stmts,
   10077              :                                      MINUS_EXPR, step_vectype,
   10078              :                                      step_mul, peel_mul);
   10079              :         }
   10080              : 
   10081              :       /* Create the induction-phi that defines the induction-operand.  */
   10082        16667 :       vec_dest = vect_get_new_vect_var (vectype, vect_simple_var,
   10083              :                                         "vec_iv_");
   10084        16667 :       induction_phi = create_phi_node (vec_dest, iv_loop->header);
   10085        16667 :       induc_def = PHI_RESULT (induction_phi);
   10086              : 
   10087              :       /* Create the iv update inside the loop  */
   10088        16667 :       tree up = vec_step;
   10089        16667 :       if (lupdate_mul)
   10090              :          {
   10091        16426 :            if (lupdate_mul_stmts)
   10092            0 :              gimple_seq_add_seq (&stmts, lupdate_mul_stmts);
   10093        16426 :            up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step,
   10094              :                               lupdate_mul);
   10095              :          }
   10096        16667 :       vec_def = gimple_convert (&stmts, step_vectype, induc_def);
   10097        16667 :       vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
   10098        16667 :       vec_def = gimple_convert (&stmts, vectype, vec_def);
   10099        16667 :       insert_iv_increment (&incr_si, insert_after, stmts);
   10100        16667 :       add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
   10101              :                    UNKNOWN_LOCATION);
   10102              : 
   10103        16667 :       if (init_node)
   10104          241 :         vec_init = vect_get_slp_vect_def (init_node, ivn);
   10105        16667 :       if (!nested_in_vect_loop
   10106        16667 :           && step_mul
   10107        16667 :           && !integer_zerop (step_mul))
   10108              :         {
   10109        15956 :           gcc_assert (invariant);
   10110        15956 :           vec_def = gimple_convert (&init_stmts, step_vectype, vec_init);
   10111        15956 :           up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
   10112              :                              vec_step, step_mul);
   10113        15956 :           vec_def = gimple_build (&init_stmts, PLUS_EXPR, step_vectype,
   10114              :                                   vec_def, up);
   10115        15956 :           vec_init = gimple_convert (&init_stmts, vectype, vec_def);
   10116              :         }
   10117              : 
   10118              :       /* Set the arguments of the phi node:  */
   10119        16667 :       add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
   10120              : 
   10121        16667 :       slp_node->push_vec_def (induction_phi);
   10122              :     }
   10123        16079 :   if (!nested_in_vect_loop)
   10124              :     {
   10125              :       /* Fill up to the number of vectors we need for the whole group.  */
   10126        15872 :       if (nunits.is_constant (&const_nunits)
   10127        15872 :           && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
   10128        15872 :         nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
   10129              :       else
   10130              :         nivs = 1;
   10131        15872 :       vec_steps.reserve (nivs-ivn);
   10132        15872 :       unsigned generated_nivs = ivn;
   10133        15872 :       gcc_assert (generated_nivs > 0);
   10134        15908 :       for (; ivn < nivs; ++ivn)
   10135              :         {
   10136           36 :           unsigned reuse_ivn = ivn % generated_nivs;
   10137           36 :           slp_node->push_vec_def (SLP_TREE_VEC_DEFS (slp_node)[reuse_ivn]);
   10138           36 :           vec_steps.quick_push (vec_steps[reuse_ivn]);
   10139              :         }
   10140              :     }
   10141              : 
   10142              :   /* Re-use IVs when we can.  We are generating further vector
   10143              :      stmts by adding VF' * stride to the IVs generated above.  */
   10144        16079 :   if (ivn < nvects)
   10145              :     {
   10146         3550 :       if (nunits.is_constant (&const_nunits)
   10147         3550 :           && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
   10148              :         {
   10149         3550 :           unsigned vfp = (least_common_multiple (group_size, const_nunits)
   10150         3550 :                           / group_size);
   10151         3550 :           lupdate_mul
   10152         3550 :               = build_vector_from_val (step_vectype,
   10153         3550 :                                        SCALAR_FLOAT_TYPE_P (stept)
   10154            8 :                                        ? build_real_from_wide (stept,
   10155            8 :                                                                vfp, UNSIGNED)
   10156         7092 :                                        : build_int_cstu (stept, vfp));
   10157              :         }
   10158              :       else
   10159              :         {
   10160            0 :           if (SCALAR_FLOAT_TYPE_P (stept))
   10161              :             {
   10162            0 :               tree tem = build_int_cst (integer_type_node, nunits);
   10163            0 :               lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem);
   10164              :             }
   10165              :           else
   10166            0 :             lupdate_mul = build_int_cst (stept, nunits);
   10167            0 :           lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype,
   10168              :                                                       lupdate_mul);
   10169              :         }
   10170        11498 :       for (; ivn < nvects; ++ivn)
   10171              :         {
   10172         7948 :           gimple *iv
   10173         7948 :             = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node)[ivn - nivs]);
   10174         7948 :           tree def = gimple_get_lhs (iv);
   10175         7948 :           if (ivn < 2*nivs)
   10176         3648 :             vec_steps[ivn - nivs]
   10177         3648 :               = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
   10178         3648 :                               vec_steps[ivn - nivs], lupdate_mul);
   10179         7948 :           gimple_seq stmts = NULL;
   10180         7948 :           def = gimple_convert (&stmts, step_vectype, def);
   10181        23844 :           def = gimple_build (&stmts, PLUS_EXPR, step_vectype,
   10182         7948 :                               def, vec_steps[ivn % nivs]);
   10183         7948 :           def = gimple_convert (&stmts, vectype, def);
   10184         7948 :           if (gimple_code (iv) == GIMPLE_PHI)
   10185         3648 :             gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
   10186              :           else
   10187              :             {
   10188         4300 :               gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
   10189         4300 :               gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING);
   10190              :             }
   10191         7948 :           slp_node->push_vec_def (def);
   10192              :         }
   10193              :     }
   10194              : 
   10195        16079 :   new_bb = gsi_insert_seq_on_edge_immediate (pe, init_stmts);
   10196        16079 :   gcc_assert (!new_bb);
   10197              : 
   10198        16079 :   return true;
   10199        16079 : }
   10200              : 
   10201              : /* Function vectorizable_live_operation_1.
   10202              : 
   10203              :    helper function for vectorizable_live_operation.  */
   10204              : 
   10205              : static tree
   10206         2799 : vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
   10207              :                                tree vectype, slp_tree slp_node,
   10208              :                                tree bitsize, tree bitstart, tree vec_lhs,
   10209              :                                tree lhs_type, gimple_stmt_iterator *exit_gsi)
   10210              : {
   10211         2799 :   gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
   10212              : 
   10213         2799 :   tree vec_lhs_phi = copy_ssa_name (vec_lhs);
   10214         2799 :   gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
   10215         8399 :   for (unsigned i = 0; i < gimple_phi_num_args (phi); i++)
   10216         2801 :     SET_PHI_ARG_DEF (phi, i, vec_lhs);
   10217              : 
   10218         2799 :   gimple_seq stmts = NULL;
   10219         2799 :   tree new_tree;
   10220              : 
   10221              :   /* If bitstart is 0 then we can use a BIT_FIELD_REF  */
   10222         2799 :   if (integer_zerop (bitstart))
   10223              :     {
   10224          212 :       tree scalar_res = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
   10225              :                                       vec_lhs_phi, bitsize, bitstart);
   10226              : 
   10227              :       /* Convert the extracted vector element to the scalar type.  */
   10228          212 :       new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
   10229              :     }
   10230         2587 :   else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
   10231              :     {
   10232              :       /* Emit:
   10233              : 
   10234              :          SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>
   10235              : 
   10236              :          where VEC_LHS is the vectorized live-out result, LEN is the length of
   10237              :          the vector, BIAS is the load-store bias.  The bias should not be used
   10238              :          at all since we are not using load/store operations, but LEN will be
   10239              :          REALLEN + BIAS, so subtract it to get to the correct position.  */
   10240            0 :       gcc_assert (SLP_TREE_LANES (slp_node) == 1);
   10241            0 :       gimple_seq tem = NULL;
   10242            0 :       gimple_stmt_iterator gsi = gsi_last (tem);
   10243            0 :       tree len = vect_get_loop_len (loop_vinfo, &gsi,
   10244              :                                     &LOOP_VINFO_LENS (loop_vinfo),
   10245              :                                     1, vectype, 0, 1, false);
   10246            0 :       gimple_seq_add_seq (&stmts, tem);
   10247              : 
   10248              :       /* LAST_INDEX = LEN - 1.  */
   10249            0 :       tree last_index = gimple_build (&stmts, MINUS_EXPR, TREE_TYPE (len),
   10250            0 :                                      len, build_one_cst (TREE_TYPE (len)));
   10251              : 
   10252              :       /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>.  */
   10253            0 :       tree scalar_res
   10254            0 :         = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
   10255              :                         vec_lhs_phi, last_index);
   10256              : 
   10257              :       /* Convert the extracted vector element to the scalar type.  */
   10258            0 :       new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
   10259              :     }
   10260         2587 :   else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
   10261              :     {
   10262              :       /* Emit:
   10263              : 
   10264              :          SCALAR_RES = EXTRACT_LAST_ACTIVE <VEC_LHS, MASK>
   10265              : 
   10266              :          where VEC_LHS is the vectorized live-out result and MASK is
   10267              :          the loop mask for the final iteration.  */
   10268            0 :       gcc_assert (SLP_TREE_LANES (slp_node) == 1);
   10269            0 :       tree scalar_type = TREE_TYPE (vectype);
   10270            0 :       gimple_seq tem = NULL;
   10271            0 :       gimple_stmt_iterator gsi = gsi_last (tem);
   10272            0 :       tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
   10273              :                                       &LOOP_VINFO_MASKS (loop_vinfo),
   10274              :                                       1, vectype, 0);
   10275            0 :       tree scalar_res;
   10276            0 :       gimple_seq_add_seq (&stmts, tem);
   10277              : 
   10278            0 :       scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST_ACTIVE, scalar_type,
   10279              :                                  mask, vec_lhs_phi);
   10280              : 
   10281              :       /* Convert the extracted vector element to the scalar type.  */
   10282            0 :       new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
   10283              :     }
   10284              :   else
   10285              :     {
   10286         2587 :       tree bftype = TREE_TYPE (vectype);
   10287         2587 :       if (VECTOR_BOOLEAN_TYPE_P (vectype))
   10288           85 :         bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
   10289         2587 :       new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart);
   10290         2587 :       new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
   10291              :                                        &stmts, true, NULL_TREE);
   10292              :     }
   10293              : 
   10294         2799 :   *exit_gsi = gsi_after_labels (exit_bb);
   10295         2799 :   if (stmts)
   10296         2799 :     gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT);
   10297              : 
   10298         2799 :   return new_tree;
   10299              : }
   10300              : 
   10301              : /* Function vectorizable_live_operation.
   10302              : 
   10303              :    STMT_INFO computes a value that is used outside the loop.  Check if
   10304              :    it can be supported.  */
   10305              : 
   10306              : bool
   10307       287949 : vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   10308              :                              slp_tree slp_node, slp_instance slp_node_instance,
   10309              :                              int slp_index, bool vec_stmt_p,
   10310              :                              stmt_vector_for_cost *cost_vec)
   10311              : {
   10312       287949 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   10313       287949 :   imm_use_iterator imm_iter;
   10314       287949 :   tree lhs, lhs_type, bitsize;
   10315       287949 :   tree vectype = SLP_TREE_VECTYPE (slp_node);
   10316       287949 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
   10317       287949 :   gimple *use_stmt;
   10318       287949 :   use_operand_p use_p;
   10319       287949 :   auto_vec<tree> vec_oprnds;
   10320       287949 :   int vec_entry = 0;
   10321       287949 :   poly_uint64 vec_index = 0;
   10322              : 
   10323       287949 :   gcc_assert (STMT_VINFO_LIVE_P (stmt_info)
   10324              :               || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
   10325              : 
   10326              :   /* If a stmt of a reduction is live, vectorize it via
   10327              :      vect_create_epilog_for_reduction.  vectorizable_reduction assessed
   10328              :      validity so just trigger the transform here.  */
   10329       287949 :   if (vect_is_reduction (slp_node))
   10330              :     {
   10331        84725 :       if (!vec_stmt_p)
   10332              :         {
   10333        61197 :           SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
   10334        61197 :           return true;
   10335              :         }
   10336              :       /* For SLP reductions we vectorize the epilogue for all involved stmts
   10337              :          together.  For SLP reduction chains we only get here once.  */
   10338        23528 :       if (SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_group
   10339        23229 :           && slp_index != 0)
   10340              :         return true;
   10341        23048 :       vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
   10342        23048 :       if (VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
   10343        23048 :           || VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
   10344              :         return true;
   10345              : 
   10346        22071 :       if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
   10347        22071 :           || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
   10348        22062 :         vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
   10349              :                                           slp_node_instance,
   10350              :                                           LOOP_VINFO_MAIN_EXIT (loop_vinfo));
   10351              : 
   10352              :       /* If early break we only have to materialize the reduction on the merge
   10353              :          block, but we have to find an alternate exit first.  */
   10354        22071 :       if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
   10355              :         {
   10356           28 :           slp_tree phis_node = slp_node_instance->reduc_phis;
   10357           28 :           stmt_info = SLP_TREE_REPRESENTATIVE (phis_node);
   10358           89 :           for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
   10359           28 :             if (exit != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
   10360              :               {
   10361           23 :                 vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
   10362              :                                                   phis_node, slp_node_instance,
   10363              :                                                   exit);
   10364           23 :                 break;
   10365           28 :               }
   10366           28 :           if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
   10367            9 :             vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
   10368              :                                               phis_node, slp_node_instance,
   10369              :                                               LOOP_VINFO_MAIN_EXIT
   10370              :                                               (loop_vinfo));
   10371              :         }
   10372              : 
   10373              :       return true;
   10374              :     }
   10375              : 
   10376              :   /* If STMT is not relevant and it is a simple assignment and its inputs are
   10377              :      invariant then it can remain in place, unvectorized.  The original last
   10378              :      scalar value that it computes will be used.  */
   10379       203224 :   if (!STMT_VINFO_RELEVANT_P (stmt_info))
   10380              :     {
   10381            0 :       gcc_assert (is_simple_and_all_uses_invariant (stmt_info, loop_vinfo));
   10382            0 :       if (dump_enabled_p ())
   10383            0 :         dump_printf_loc (MSG_NOTE, vect_location,
   10384              :                          "statement is simple and uses invariant.  Leaving in "
   10385              :                          "place.\n");
   10386              :       return true;
   10387              :     }
   10388              : 
   10389       203224 :   gcc_assert (slp_index >= 0);
   10390              : 
   10391       203224 :   int num_vec = vect_get_num_copies (vinfo, slp_node);
   10392       203224 :   poly_uint64 pos = slp_index;
   10393       203224 :   if (loop_vinfo)
   10394              :     {
   10395              :       /* Get the last occurrence of the scalar index from the concatenation of
   10396              :          all the slp vectors. Calculate which slp vector it is and the index
   10397              :          within.  */
   10398        80182 :       int num_scalar = SLP_TREE_LANES (slp_node);
   10399        80182 :       pos += (num_vec * nunits) - num_scalar;
   10400              :     }
   10401              : 
   10402              :   /* Calculate which vector contains the result, and which lane of
   10403              :      that vector we need.  */
   10404       203224 :   if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
   10405              :     {
   10406              :       if (dump_enabled_p ())
   10407              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10408              :                          "Cannot determine which vector holds the"
   10409              :                          " final result.\n");
   10410              :       return false;
   10411              :     }
   10412              : 
   10413       203224 :   if (!vec_stmt_p)
   10414              :     {
   10415              :       /* No transformation required.  */
   10416       186370 :       if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
   10417              :         {
   10418        31920 :           if (SLP_TREE_LANES (slp_node) != 1)
   10419              :             {
   10420           19 :               if (dump_enabled_p ())
   10421           19 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10422              :                                  "can't operate on partial vectors "
   10423              :                                  "because an SLP statement is live after "
   10424              :                                  "the loop.\n");
   10425           19 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
   10426              :             }
   10427        31901 :           else if (num_vec > 1)
   10428              :             {
   10429        18647 :               if (dump_enabled_p ())
   10430           69 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10431              :                                  "can't operate on partial vectors "
   10432              :                                  "because ncopies is greater than 1.\n");
   10433        18647 :               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
   10434              :             }
   10435              :           else
   10436              :             {
   10437        13254 :               if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST_ACTIVE, vectype,
   10438              :                                                   OPTIMIZE_FOR_SPEED))
   10439            0 :                 vect_record_loop_mask (loop_vinfo,
   10440              :                                        &LOOP_VINFO_MASKS (loop_vinfo),
   10441              :                                        1, vectype, NULL);
   10442        13254 :               else if (can_vec_extract_var_idx_p (
   10443        13254 :                          TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
   10444            0 :                 vect_record_loop_len (loop_vinfo,
   10445              :                                       &LOOP_VINFO_LENS (loop_vinfo),
   10446              :                                       1, vectype, 1);
   10447              :               else
   10448              :                 {
   10449        13254 :                   if (dump_enabled_p ())
   10450          752 :                     dump_printf_loc (
   10451          752 :                       MSG_MISSED_OPTIMIZATION, vect_location,
   10452              :                       "can't operate on partial vectors "
   10453              :                       "because the target doesn't support extract "
   10454              :                       "last reduction.\n");
   10455        13254 :                   LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
   10456              :                 }
   10457              :             }
   10458              :         }
   10459        31920 :       record_stmt_cost (cost_vec, 1, vec_to_scalar, slp_node,
   10460              :                         0, loop_vinfo ? vect_epilogue : vect_body);
   10461       186370 :       SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
   10462       186370 :       return true;
   10463              :     }
   10464              : 
   10465              :   /* Use the lhs of the original scalar statement.  */
   10466        16854 :   gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
   10467        16854 :   if (dump_enabled_p ())
   10468         1048 :     dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
   10469              :                      "stmt %G", stmt);
   10470              : 
   10471        16854 :   lhs = gimple_get_lhs (stmt);
   10472        16854 :   lhs_type = TREE_TYPE (lhs);
   10473              : 
   10474        16854 :   bitsize = vector_element_bits_tree (vectype);
   10475              : 
   10476              :   /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
   10477        16854 :   gcc_assert (!loop_vinfo
   10478              :               || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
   10479              :                    && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
   10480              :                   || SLP_TREE_LANES (slp_node) == 1));
   10481              : 
   10482              :   /* Get the correct slp vectorized stmt.  */
   10483        16854 :   tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
   10484              : 
   10485              :   /* In case we need to early break vectorize also get the first stmt.  */
   10486        16854 :   tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
   10487              : 
   10488              :   /* Get entry to use.  */
   10489        16854 :   tree bitstart = bitsize_int (vec_index);
   10490        16854 :   bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
   10491              : 
   10492        16854 :   if (loop_vinfo)
   10493              :     {
   10494              :       /* Ensure the VEC_LHS for lane extraction stmts satisfy loop-closed PHI
   10495              :          requirement, insert one phi node for it.  It looks like:
   10496              :            loop;
   10497              :          BB:
   10498              :            # lhs' = PHI <lhs>
   10499              :          ==>
   10500              :            loop;
   10501              :          BB:
   10502              :            # vec_lhs' = PHI <vec_lhs>
   10503              :            new_tree = lane_extract <vec_lhs', ...>;
   10504              :            lhs' = new_tree;  */
   10505              : 
   10506         2882 :       class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   10507              :       /* Check if we have a loop where the chosen exit is not the main exit,
   10508              :          in these cases for an early break we restart the iteration the vector code
   10509              :          did.  For the live values we want the value at the start of the iteration
   10510              :          rather than at the end.  */
   10511         2882 :       edge main_e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
   10512         2882 :       bool all_exits_as_early_p = LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo);
   10513        12106 :       FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
   10514         9224 :         if (!is_gimple_debug (use_stmt)
   10515         9224 :             && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
   10516         2799 :           FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
   10517              :             {
   10518         2799 :               edge e = gimple_phi_arg_edge (as_a <gphi *> (use_stmt),
   10519         2799 :                                            phi_arg_index_from_use (use_p));
   10520         2799 :               gcc_assert (loop_exit_edge_p (loop, e));
   10521         2799 :               bool main_exit_edge = e == main_e;
   10522         2799 :               tree tmp_vec_lhs = vec_lhs;
   10523         2799 :               tree tmp_bitstart = bitstart;
   10524              : 
   10525              :               /* For early exit where the exit is not in the BB that leads
   10526              :                  to the latch then we're restarting the iteration in the
   10527              :                  scalar loop.  So get the first live value.  */
   10528         2799 :               bool early_break_first_element_p
   10529         2799 :                 = all_exits_as_early_p || !main_exit_edge;
   10530         2799 :               if (early_break_first_element_p)
   10531              :                 {
   10532          194 :                   tmp_vec_lhs = vec_lhs0;
   10533          194 :                   tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
   10534              :                 }
   10535              : 
   10536         2799 :               gimple_stmt_iterator exit_gsi;
   10537         2799 :               tree new_tree
   10538         2799 :                   = vectorizable_live_operation_1 (loop_vinfo,
   10539              :                                                    e->dest, vectype,
   10540              :                                                    slp_node, bitsize,
   10541              :                                                    tmp_bitstart, tmp_vec_lhs,
   10542              :                                                    lhs_type, &exit_gsi);
   10543              : 
   10544         2799 :               auto gsi = gsi_for_stmt (use_stmt);
   10545         2799 :               tree lhs_phi = gimple_phi_result (use_stmt);
   10546         2799 :               remove_phi_node (&gsi, false);
   10547         2799 :               gimple *copy = gimple_build_assign (lhs_phi, new_tree);
   10548         2799 :               gsi_insert_before (&exit_gsi, copy, GSI_SAME_STMT);
   10549         2799 :               break;
   10550         2882 :             }
   10551              : 
   10552              :       /* There a no further out-of-loop uses of lhs by LC-SSA construction.  */
   10553         9307 :       FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
   10554         6425 :         gcc_assert (is_gimple_debug (use_stmt)
   10555         2882 :                     || flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)));
   10556              :     }
   10557              :   else
   10558              :     {
   10559              :       /* For basic-block vectorization simply insert the lane-extraction.  */
   10560        13972 :       tree bftype = TREE_TYPE (vectype);
   10561        13972 :       if (VECTOR_BOOLEAN_TYPE_P (vectype))
   10562           18 :         bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
   10563        13972 :       tree new_tree = build3 (BIT_FIELD_REF, bftype,
   10564              :                               vec_lhs, bitsize, bitstart);
   10565        13972 :       gimple_seq stmts = NULL;
   10566        13972 :       new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
   10567              :                                        &stmts, true, NULL_TREE);
   10568        13972 :       if (TREE_CODE (new_tree) == SSA_NAME
   10569        27944 :           && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
   10570            0 :         SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = 1;
   10571        13972 :       gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
   10572        13972 :       if (TREE_CODE (vec_lhs) != SSA_NAME
   10573        13972 :           || SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
   10574        13972 :           || (vinfo->bbs[0] != gimple_bb (vec_stmt)
   10575         9745 :               && dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
   10576         9745 :                                  gimple_bb (vec_stmt))))
   10577            2 :         vinfo->insert_seq_on_entry (stmt_info, stmts);
   10578        13970 :       else if (is_a <gphi *> (vec_stmt))
   10579              :         {
   10580         1384 :           gimple_stmt_iterator si = gsi_after_labels (gimple_bb (vec_stmt));
   10581         1384 :           gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
   10582              :         }
   10583              :       else
   10584              :         {
   10585        12586 :           gimple_stmt_iterator si = gsi_for_stmt (vec_stmt);
   10586        12586 :           gsi_insert_seq_after (&si, stmts, GSI_SAME_STMT);
   10587              :         }
   10588              : 
   10589              :       /* Replace use of lhs with newly computed result.  If the use stmt is a
   10590              :          single arg PHI, just replace all uses of PHI result.  It's necessary
   10591              :          because lcssa PHI defining lhs may be before newly inserted stmt.  */
   10592        13972 :       use_operand_p use_p;
   10593        13972 :       stmt_vec_info use_stmt_info;
   10594        76780 :       FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
   10595        62808 :         if (!is_gimple_debug (use_stmt)
   10596        62808 :             && (!(use_stmt_info = vinfo->lookup_stmt (use_stmt))
   10597        46962 :                 || !PURE_SLP_STMT (use_stmt_info)))
   10598              :           {
   10599              :             /* ???  This can happen when the live lane ends up being
   10600              :                rooted in a vector construction code-generated by an
   10601              :                external SLP node (and code-generation for that already
   10602              :                happened).
   10603              :                Doing this is what would happen if that vector CTOR
   10604              :                were not code-generated yet so it is not too bad.
   10605              :                ???  In fact we'd likely want to avoid this situation
   10606              :                in the first place.  */
   10607        25801 :             if (TREE_CODE (new_tree) == SSA_NAME
   10608        25801 :                 && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
   10609        25801 :                 && gimple_code (use_stmt) != GIMPLE_PHI
   10610        48463 :                 && !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (new_tree),
   10611              :                                                 use_stmt))
   10612              :               {
   10613            0 :                 if (dump_enabled_p ())
   10614            0 :                   dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10615              :                                    "Using original scalar computation for "
   10616              :                                    "live lane because use precedes vector "
   10617              :                                    "def\n");
   10618            0 :                 continue;
   10619              :               }
   10620        54070 :             FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
   10621              :               {
   10622              :                 /* ???  It can also happen that we end up pulling a def into
   10623              :                    a loop where replacing out-of-loop uses would require
   10624              :                    a new LC SSA PHI node.  Retain the original scalar in
   10625              :                    those cases as well.  PR98064.  */
   10626        27035 :                 edge e;
   10627        27035 :                 if (TREE_CODE (new_tree) == SSA_NAME
   10628        27035 :                     && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
   10629        27035 :                     && TREE_CODE (vec_lhs) == SSA_NAME
   10630        27035 :                     && !SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
   10631        27035 :                     && (gimple_bb (use_stmt)->loop_father
   10632        27035 :                         != gimple_bb (vec_stmt)->loop_father)
   10633              :                     /* But a replacement in a LC PHI is OK.  This happens
   10634              :                        in gcc.dg/vect/bb-slp-57.c for example.  */
   10635         3054 :                     && (gimple_code (use_stmt) != GIMPLE_PHI
   10636         1207 :                         || (((e = phi_arg_edge_from_use (use_p)), true)
   10637         1207 :                             && !loop_exit_edge_p
   10638         1207 :                                   (gimple_bb (vec_stmt)->loop_father, e)))
   10639        29612 :                     && !flow_loop_nested_p (gimple_bb (vec_stmt)->loop_father,
   10640         2577 :                                             gimple_bb (use_stmt)->loop_father))
   10641              :                   {
   10642            0 :                     if (dump_enabled_p ())
   10643            0 :                       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
   10644              :                                        "Using original scalar computation for "
   10645              :                                        "live lane because there is an "
   10646              :                                        "out-of-loop definition for it\n");
   10647            0 :                     continue;
   10648              :                   }
   10649        27035 :                 SET_USE (use_p, new_tree);
   10650              :               }
   10651        25801 :             update_stmt (use_stmt);
   10652        13972 :           }
   10653              :     }
   10654              : 
   10655              :   return true;
   10656       287949 : }
   10657              : 
   10658              : /* Given loop represented by LOOP_VINFO, return true if computation of
   10659              :    LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
   10660              :    otherwise.  */
   10661              : 
   10662              : static bool
   10663        64347 : loop_niters_no_overflow (loop_vec_info loop_vinfo)
   10664              : {
   10665        64347 :   gcc_assert (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo));
   10666              : 
   10667              :   /* Constant case.  */
   10668        64347 :   if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
   10669              :     {
   10670        37967 :       tree cst_niters = LOOP_VINFO_NITERS (loop_vinfo);
   10671        37967 :       tree cst_nitersm1 = LOOP_VINFO_NITERSM1 (loop_vinfo);
   10672              : 
   10673        37967 :       gcc_assert (TREE_CODE (cst_niters) == INTEGER_CST);
   10674        37967 :       gcc_assert (TREE_CODE (cst_nitersm1) == INTEGER_CST);
   10675        37967 :       if (wi::to_widest (cst_nitersm1) < wi::to_widest (cst_niters))
   10676              :         return true;
   10677              :     }
   10678              : 
   10679        26380 :   widest_int max;
   10680        26380 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   10681              :   /* Check the upper bound of loop niters.  */
   10682        26380 :   if (get_max_loop_iterations (loop, &max))
   10683              :     {
   10684        26380 :       tree type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
   10685        26380 :       signop sgn = TYPE_SIGN (type);
   10686        26380 :       widest_int type_max = widest_int::from (wi::max_value (type), sgn);
   10687        26380 :       if (max < type_max)
   10688        26175 :         return true;
   10689        26380 :     }
   10690              :   return false;
   10691        64347 : }
   10692              : 
   10693              : /* Return a mask type with half the number of elements as OLD_TYPE,
   10694              :    given that it should have mode NEW_MODE.  */
   10695              : 
   10696              : tree
   10697         4699 : vect_halve_mask_nunits (tree old_type, machine_mode new_mode)
   10698              : {
   10699         4699 :   poly_uint64 nunits = exact_div (TYPE_VECTOR_SUBPARTS (old_type), 2);
   10700         4699 :   return build_truth_vector_type_for_mode (nunits, new_mode);
   10701              : }
   10702              : 
   10703              : /* Return a mask type with twice as many elements as OLD_TYPE,
   10704              :    given that it should have mode NEW_MODE.  */
   10705              : 
   10706              : tree
   10707         7028 : vect_double_mask_nunits (tree old_type, machine_mode new_mode)
   10708              : {
   10709         7028 :   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (old_type) * 2;
   10710         7028 :   return build_truth_vector_type_for_mode (nunits, new_mode);
   10711              : }
   10712              : 
   10713              : /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
   10714              :    contain a sequence of NVECTORS masks that each control a vector of type
   10715              :    VECTYPE.  If SCALAR_MASK is nonnull, the fully-masked loop would AND
   10716              :    these vector masks with the vector version of SCALAR_MASK.  */
   10717              : 
   10718              : void
   10719       107752 : vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
   10720              :                        unsigned int nvectors, tree vectype, tree scalar_mask)
   10721              : {
   10722       107752 :   gcc_assert (nvectors != 0);
   10723              : 
   10724       107752 :   if (scalar_mask)
   10725              :     {
   10726         5051 :       scalar_cond_masked_key cond (scalar_mask, nvectors);
   10727         5051 :       loop_vinfo->scalar_cond_masked_set.add (cond);
   10728              :     }
   10729              : 
   10730       107752 :   masks->mask_set.add (std::make_pair (vectype, nvectors));
   10731       107752 : }
   10732              : 
   10733              : /* Given a complete set of masks MASKS, extract mask number INDEX
   10734              :    for an rgroup that operates on NVECTORS vectors of type VECTYPE,
   10735              :    where 0 <= INDEX < NVECTORS.  Insert any set-up statements before GSI.
   10736              : 
   10737              :    See the comment above vec_loop_masks for more details about the mask
   10738              :    arrangement.  */
   10739              : 
   10740              : tree
   10741          232 : vect_get_loop_mask (loop_vec_info loop_vinfo,
   10742              :                     gimple_stmt_iterator *gsi, vec_loop_masks *masks,
   10743              :                     unsigned int nvectors, tree vectype, unsigned int index)
   10744              : {
   10745          232 :   if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
   10746              :       == vect_partial_vectors_while_ult)
   10747              :     {
   10748            0 :       rgroup_controls *rgm = &(masks->rgc_vec)[nvectors - 1];
   10749            0 :       tree mask_type = rgm->type;
   10750              : 
   10751              :       /* Populate the rgroup's mask array, if this is the first time we've
   10752              :          used it.  */
   10753            0 :       if (rgm->controls.is_empty ())
   10754              :         {
   10755            0 :           rgm->controls.safe_grow_cleared (nvectors, true);
   10756            0 :           for (unsigned int i = 0; i < nvectors; ++i)
   10757              :             {
   10758            0 :               tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
   10759              :               /* Provide a dummy definition until the real one is available.  */
   10760            0 :               SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
   10761            0 :               rgm->controls[i] = mask;
   10762              :             }
   10763              :         }
   10764              : 
   10765            0 :       tree mask = rgm->controls[index];
   10766            0 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
   10767            0 :                     TYPE_VECTOR_SUBPARTS (vectype)))
   10768              :         {
   10769              :           /* A loop mask for data type X can be reused for data type Y
   10770              :              if X has N times more elements than Y and if Y's elements
   10771              :              are N times bigger than X's.  In this case each sequence
   10772              :              of N elements in the loop mask will be all-zero or all-one.
   10773              :              We can then view-convert the mask so that each sequence of
   10774              :              N elements is replaced by a single element.  */
   10775            0 :           gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type),
   10776              :                                   TYPE_VECTOR_SUBPARTS (vectype)));
   10777            0 :           gimple_seq seq = NULL;
   10778            0 :           mask_type = truth_type_for (vectype);
   10779            0 :           mask = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, mask);
   10780            0 :           if (seq)
   10781            0 :             gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
   10782              :         }
   10783              :       return mask;
   10784              :     }
   10785          232 :   else if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
   10786              :            == vect_partial_vectors_avx512)
   10787              :     {
   10788              :       /* The number of scalars per iteration and the number of vectors are
   10789              :          both compile-time constants.  */
   10790          232 :       unsigned int nscalars_per_iter
   10791          232 :         = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
   10792          232 :                      LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
   10793              : 
   10794          232 :       rgroup_controls *rgm = &masks->rgc_vec[nscalars_per_iter - 1];
   10795              : 
   10796              :       /* The stored nV is dependent on the mask type produced.  */
   10797          232 :       gcc_assert (exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
   10798              :                              TYPE_VECTOR_SUBPARTS (rgm->type)).to_constant ()
   10799              :                   == rgm->factor);
   10800          232 :       nvectors = rgm->factor;
   10801              : 
   10802              :       /* Populate the rgroup's mask array, if this is the first time we've
   10803              :          used it.  */
   10804          232 :       if (rgm->controls.is_empty ())
   10805              :         {
   10806           26 :           rgm->controls.safe_grow_cleared (nvectors, true);
   10807          144 :           for (unsigned int i = 0; i < nvectors; ++i)
   10808              :             {
   10809           92 :               tree mask = make_temp_ssa_name (rgm->type, NULL, "loop_mask");
   10810              :               /* Provide a dummy definition until the real one is available.  */
   10811           92 :               SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
   10812           92 :               rgm->controls[i] = mask;
   10813              :             }
   10814              :         }
   10815          232 :       if (known_eq (TYPE_VECTOR_SUBPARTS (rgm->type),
   10816              :                     TYPE_VECTOR_SUBPARTS (vectype)))
   10817          170 :         return rgm->controls[index];
   10818              : 
   10819              :       /* Split the vector if needed.  Since we are dealing with integer mode
   10820              :          masks with AVX512 we can operate on the integer representation
   10821              :          performing the whole vector shifting.  */
   10822           62 :       unsigned HOST_WIDE_INT factor;
   10823           62 :       bool ok = constant_multiple_p (TYPE_VECTOR_SUBPARTS (rgm->type),
   10824           62 :                                      TYPE_VECTOR_SUBPARTS (vectype), &factor);
   10825            0 :       gcc_assert (ok);
   10826           62 :       gcc_assert (GET_MODE_CLASS (TYPE_MODE (rgm->type)) == MODE_INT);
   10827           62 :       tree mask_type = truth_type_for (vectype);
   10828           62 :       gcc_assert (GET_MODE_CLASS (TYPE_MODE (mask_type)) == MODE_INT);
   10829           62 :       unsigned vi = index / factor;
   10830           62 :       unsigned vpart = index % factor;
   10831           62 :       tree vec = rgm->controls[vi];
   10832           62 :       gimple_seq seq = NULL;
   10833           62 :       vec = gimple_build (&seq, VIEW_CONVERT_EXPR,
   10834           62 :                           lang_hooks.types.type_for_mode
   10835           62 :                                 (TYPE_MODE (rgm->type), 1), vec);
   10836              :       /* For integer mode masks simply shift the right bits into position.  */
   10837           62 :       if (vpart != 0)
   10838           47 :         vec = gimple_build (&seq, RSHIFT_EXPR, TREE_TYPE (vec), vec,
   10839              :                             build_int_cst (integer_type_node,
   10840           94 :                                            (TYPE_VECTOR_SUBPARTS (vectype)
   10841           47 :                                             * vpart)));
   10842           62 :       vec = gimple_convert (&seq, lang_hooks.types.type_for_mode
   10843           62 :                                     (TYPE_MODE (mask_type), 1), vec);
   10844           62 :       vec = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, vec);
   10845           62 :       if (seq)
   10846           62 :         gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
   10847              :       return vec;
   10848              :     }
   10849              :   else
   10850            0 :     gcc_unreachable ();
   10851              : }
   10852              : 
   10853              : /* Record that LOOP_VINFO would need LENS to contain a sequence of NVECTORS
   10854              :    lengths for controlling an operation on VECTYPE.  The operation splits
   10855              :    each element of VECTYPE into FACTOR separate subelements, measuring the
   10856              :    length as a number of these subelements.  */
   10857              : 
   10858              : void
   10859            0 : vect_record_loop_len (loop_vec_info loop_vinfo, vec_loop_lens *lens,
   10860              :                       unsigned int nvectors, tree vectype, unsigned int factor)
   10861              : {
   10862            0 :   gcc_assert (nvectors != 0);
   10863            0 :   if (lens->length () < nvectors)
   10864            0 :     lens->safe_grow_cleared (nvectors, true);
   10865            0 :   rgroup_controls *rgl = &(*lens)[nvectors - 1];
   10866              : 
   10867              :   /* The number of scalars per iteration, scalar occupied bytes and
   10868              :      the number of vectors are both compile-time constants.  */
   10869            0 :   unsigned int nscalars_per_iter
   10870            0 :     = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
   10871            0 :                  LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
   10872              : 
   10873            0 :   if (rgl->max_nscalars_per_iter < nscalars_per_iter)
   10874              :     {
   10875              :       /* For now, we only support cases in which all loads and stores fall back
   10876              :          to VnQI or none do.  */
   10877            0 :       gcc_assert (!rgl->max_nscalars_per_iter
   10878              :                   || (rgl->factor == 1 && factor == 1)
   10879              :                   || (rgl->max_nscalars_per_iter * rgl->factor
   10880              :                       == nscalars_per_iter * factor));
   10881            0 :       rgl->max_nscalars_per_iter = nscalars_per_iter;
   10882            0 :       rgl->type = vectype;
   10883            0 :       rgl->factor = factor;
   10884              :     }
   10885            0 : }
   10886              : 
   10887              : /* Given a complete set of lengths LENS, extract length number INDEX
   10888              :    for an rgroup that operates on NVECTORS vectors of type VECTYPE,
   10889              :    where 0 <= INDEX < NVECTORS.  Return a value that contains FACTOR
   10890              :    multiplied by the number of elements that should be processed.
   10891              :    Insert any set-up statements before GSI.  */
   10892              : 
   10893              : tree
   10894            0 : vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
   10895              :                    vec_loop_lens *lens, unsigned int nvectors, tree vectype,
   10896              :                    unsigned int index, unsigned int factor, bool adjusted)
   10897              : {
   10898            0 :   rgroup_controls *rgl = &(*lens)[nvectors - 1];
   10899            0 :   bool use_bias_adjusted_len =
   10900            0 :     LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) != 0;
   10901              : 
   10902              :   /* Populate the rgroup's len array, if this is the first time we've
   10903              :      used it.  */
   10904            0 :   if (rgl->controls.is_empty ())
   10905              :     {
   10906            0 :       rgl->controls.safe_grow_cleared (nvectors, true);
   10907            0 :       for (unsigned int i = 0; i < nvectors; ++i)
   10908              :         {
   10909            0 :           tree len_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
   10910            0 :           gcc_assert (len_type != NULL_TREE);
   10911              : 
   10912            0 :           tree len = make_temp_ssa_name (len_type, NULL, "loop_len");
   10913              : 
   10914              :           /* Provide a dummy definition until the real one is available.  */
   10915            0 :           SSA_NAME_DEF_STMT (len) = gimple_build_nop ();
   10916            0 :           rgl->controls[i] = len;
   10917              : 
   10918            0 :           if (use_bias_adjusted_len)
   10919              :             {
   10920            0 :               gcc_assert (i == 0);
   10921            0 :               tree adjusted_len =
   10922            0 :                 make_temp_ssa_name (len_type, NULL, "adjusted_loop_len");
   10923            0 :               SSA_NAME_DEF_STMT (adjusted_len) = gimple_build_nop ();
   10924            0 :               rgl->bias_adjusted_ctrl = adjusted_len;
   10925              :             }
   10926              :         }
   10927              :     }
   10928              : 
   10929            0 :   if (use_bias_adjusted_len && adjusted)
   10930            0 :     return rgl->bias_adjusted_ctrl;
   10931              : 
   10932            0 :   tree loop_len = rgl->controls[index];
   10933            0 :   if (rgl->factor == 1 && factor == 1)
   10934              :     {
   10935            0 :       poly_int64 nunits1 = TYPE_VECTOR_SUBPARTS (rgl->type);
   10936            0 :       poly_int64 nunits2 = TYPE_VECTOR_SUBPARTS (vectype);
   10937            0 :       if (maybe_ne (nunits1, nunits2))
   10938              :         {
   10939              :           /* A loop len for data type X can be reused for data type Y
   10940              :              if X has N times more elements than Y and if Y's elements
   10941              :              are N times bigger than X's.  */
   10942            0 :           gcc_assert (multiple_p (nunits1, nunits2));
   10943            0 :           factor = exact_div (nunits1, nunits2).to_constant ();
   10944            0 :           tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
   10945            0 :           gimple_seq seq = NULL;
   10946            0 :           loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
   10947            0 :                                    build_int_cst (iv_type, factor));
   10948            0 :           if (seq)
   10949            0 :             gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
   10950              :         }
   10951            0 :     }
   10952            0 :   else if (factor && rgl->factor != factor)
   10953              :     {
   10954              :       /* The number of scalars per iteration, scalar occupied bytes and
   10955              :          the number of vectors are both compile-time constants.  */
   10956            0 :       unsigned int nscalars_per_iter
   10957            0 :         = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
   10958            0 :                      LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
   10959            0 :       unsigned int rglvecsize = rgl->factor * rgl->max_nscalars_per_iter;
   10960            0 :       unsigned int vecsize = nscalars_per_iter * factor;
   10961            0 :       if (rglvecsize > vecsize)
   10962              :         {
   10963            0 :           unsigned int fac = rglvecsize / vecsize;
   10964            0 :           tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
   10965            0 :           gimple_seq seq = NULL;
   10966            0 :           loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
   10967            0 :                                    build_int_cst (iv_type, fac));
   10968            0 :           if (seq)
   10969            0 :             gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
   10970              :         }
   10971            0 :       else if (rglvecsize < vecsize)
   10972              :         {
   10973            0 :           unsigned int fac = vecsize / rglvecsize;
   10974            0 :           tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
   10975            0 :           gimple_seq seq = NULL;
   10976            0 :           loop_len = gimple_build (&seq, MULT_EXPR, iv_type, loop_len,
   10977            0 :                                    build_int_cst (iv_type, fac));
   10978            0 :           if (seq)
   10979            0 :             gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
   10980              :         }
   10981              :     }
   10982              :   return loop_len;
   10983              : }
   10984              : 
   10985              : /* Generate the tree for the loop len mask and return it.  Given the lens,
   10986              :    nvectors, vectype, index and factor to gen the len mask as below.
   10987              : 
   10988              :    tree len_mask = VCOND_MASK_LEN (compare_mask, ones, zero, len, bias)
   10989              : */
   10990              : tree
   10991            0 : vect_gen_loop_len_mask (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
   10992              :                         gimple_stmt_iterator *cond_gsi, vec_loop_lens *lens,
   10993              :                         unsigned int nvectors, tree vectype, tree stmt,
   10994              :                         unsigned int index, unsigned int factor)
   10995              : {
   10996            0 :   tree all_one_mask = build_all_ones_cst (vectype);
   10997            0 :   tree all_zero_mask = build_zero_cst (vectype);
   10998            0 :   tree len = vect_get_loop_len (loop_vinfo, gsi, lens, nvectors, vectype, index,
   10999              :                                 factor, true);
   11000            0 :   tree bias = build_int_cst (intQI_type_node,
   11001            0 :                              LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo));
   11002            0 :   tree len_mask = make_temp_ssa_name (TREE_TYPE (stmt), NULL, "vec_len_mask");
   11003            0 :   gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5, stmt,
   11004              :                                             all_one_mask, all_zero_mask, len,
   11005              :                                             bias);
   11006            0 :   gimple_call_set_lhs (call, len_mask);
   11007            0 :   gsi_insert_before (cond_gsi, call, GSI_SAME_STMT);
   11008              : 
   11009            0 :   return len_mask;
   11010              : }
   11011              : 
   11012              : /* Update the exit profile counts scaling them from scalar to vector counts.
   11013              : 
   11014              :    The calculations are as follows.
   11015              : 
   11016              :    let P be the scalar probability that one iteration takes an early exit edge.
   11017              :    1 - P is then the probabiliy that the exit is not taken.
   11018              : 
   11019              :    Since every vector iteration handles VF scalar iterations, the early exit
   11020              :    branch can be taken when any of the lanes are true.
   11021              : 
   11022              :    We invert this and first calculate the probablity that you reach the latch,
   11023              :    i.e. the probability that the vector code does not exit is the probability
   11024              :    Q that none of the lanes are true.
   11025              : 
   11026              :    Q = (1 - P) * (1 - P) * ... * (1 - P)
   11027              :      = (1 - P) ^ VF
   11028              : 
   11029              :    and so the probability that at least one lane breaks is 1 - Q,
   11030              :    or rather 1 - (i - P) ^ VF.
   11031              : 
   11032              :    For example, if the scalar break probability is 10% and VF is 4:
   11033              : 
   11034              :    P = 0.1
   11035              :    1 - P = 0.9
   11036              :    (1 - P) ^ VF = 0.9 ^ 4 = 0.6561
   11037              :    1 - (1 - p) ^ VF = 1 - 0.6561 = 0.3439
   11038              : 
   11039              :    So a scalar early-exit edge that is taken 10% of the time becomes a vector
   11040              :    early-exit edge that is taken about 34.4% of the time, because each vector
   11041              :    iteration gives the break condition four chances to trigger.  */
   11042              : 
   11043              : static void
   11044        64392 : vect_update_early_break_profiles (loop_vec_info loop_vinfo)
   11045              : {
   11046        64392 :   if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
   11047        62906 :     return;
   11048              : 
   11049         1486 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   11050         1486 :   unsigned int vf = vect_vf_for_cost (loop_vinfo);
   11051              : 
   11052         1486 :   auto_vec<std::pair<edge, profile_probability>, 8> updates;
   11053         7556 :   for (edge e : get_loop_exit_edges (loop))
   11054              :     {
   11055         4485 :       if ((e == LOOP_VINFO_MAIN_EXIT (loop_vinfo)
   11056         1432 :            && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
   11057         3143 :           || !e->probability.initialized_p ())
   11058         1387 :         continue;
   11059              : 
   11060              :       /* For any early break exits, the probablity of exiting are based on if
   11061              :          any lane is true, and so are tied to VF since you have VF chances.
   11062              :          As such scale the scalar profile by VF to the the probability for the
   11063              :          vector edges using 1 - (1 - p) ^ VF.  */
   11064         1711 :       profile_probability probability
   11065         1711 :         = profile_probability::always () - e->probability.invert ().pow (vf);
   11066         1711 :       updates.safe_push ({e, probability});
   11067         1486 :     }
   11068              : 
   11069              :   /* First update the edge counts.  */
   11070         6169 :   for (auto update : updates)
   11071         1711 :     set_edge_probability_and_rescale_others (update.first, update.second);
   11072              : 
   11073              :   /* And then only update the in loop BBs, i.e. the fall through block for
   11074              :      the early exits.  */
   11075         6169 :   for (auto update : updates)
   11076         8555 :     for (edge e : update.first->src->succs)
   11077         5133 :       if (!loop_exit_edge_p (loop, e) && single_pred_p (e->dest))
   11078         1711 :         e->dest->count = e->count ();
   11079         1486 : }
   11080              : 
   11081              : /* Scale profiling counters by estimation for LOOP which is vectorized
   11082              :    by factor VF.
   11083              :    If FLAT is true, the loop we started with had unrealistically flat
   11084              :    profile.  */
   11085              : 
   11086              : static void
   11087        64392 : scale_profile_for_vect_loop (loop_vec_info loop_vinfo,
   11088              :                              class loop *loop, edge exit_e, unsigned vf,
   11089              :                              bool flat)
   11090              : {
   11091              :   /* First scale any early exits.  */
   11092        64392 :   vect_update_early_break_profiles (loop_vinfo);
   11093              : 
   11094              :   /* For flat profiles do not scale down proportionally by VF and only
   11095              :      cap by known iteration count bounds.  */
   11096        64392 :   if (flat)
   11097              :     {
   11098        36061 :       if (dump_file && (dump_flags & TDF_DETAILS))
   11099         5626 :         fprintf (dump_file,
   11100              :                  "Vectorized loop profile seems flat; not scaling iteration "
   11101              :                  "count down by the vectorization factor %i\n", vf);
   11102        36061 :       scale_loop_profile (loop, profile_probability::always (),
   11103              :                           get_likely_max_loop_iterations_int (loop));
   11104        36061 :       return;
   11105              :     }
   11106              :   /* Loop body executes VF fewer times and exit increases VF times.  */
   11107        28331 :   profile_count entry_count = loop_preheader_edge (loop)->count ();
   11108              : 
   11109              :   /* If we have unreliable loop profile avoid dropping entry
   11110              :      count below header count.  This can happen since loops
   11111              :      has unrealistically low trip counts.  */
   11112        28331 :   while (vf > 1
   11113        29516 :          && loop->header->count > entry_count
   11114        60122 :          && loop->header->count < entry_count * vf)
   11115              :     {
   11116         2275 :       if (dump_file && (dump_flags & TDF_DETAILS))
   11117          158 :         fprintf (dump_file,
   11118              :                  "Vectorization factor %i seems too large for profile "
   11119              :                  "previously believed to be consistent; reducing.\n", vf);
   11120         2275 :       vf /= 2;
   11121              :     }
   11122              : 
   11123        28331 :   if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
   11124              :     {
   11125        28331 :       if (entry_count.nonzero_p ())
   11126        28331 :         set_edge_probability_and_rescale_others
   11127        28331 :             (exit_e,
   11128        28331 :              entry_count.probability_in (loop->header->count / vf));
   11129              :       /* Avoid producing very large exit probability when we do not have
   11130              :          sensible profile.  */
   11131            0 :       else if (exit_e->probability < profile_probability::always () / (vf * 2))
   11132            0 :         set_edge_probability_and_rescale_others (exit_e,
   11133              :                                                  exit_e->probability * vf);
   11134              :     }
   11135              : 
   11136        28331 :   loop->latch->count = single_pred_edge (loop->latch)->count ();
   11137              : 
   11138        28331 :   scale_loop_profile (loop, profile_probability::always () / vf,
   11139              :                       get_likely_max_loop_iterations_int (loop));
   11140              : }
   11141              : 
   11142              : /* Update EPILOGUE's loop_vec_info.  EPILOGUE was constructed as a copy of the
   11143              :    original loop that has now been vectorized.
   11144              : 
   11145              :    The inits of the data_references need to be advanced with the number of
   11146              :    iterations of the main loop.  This has been computed in vect_do_peeling and
   11147              :    is stored in parameter ADVANCE.
   11148              : 
   11149              :    Since the loop_vec_info of this EPILOGUE was constructed for the original
   11150              :    loop, its stmt_vec_infos all point to the original statements.  These need
   11151              :    to be updated to point to their corresponding copies.
   11152              : 
   11153              :    The data_reference's connections also need to be updated.  Their
   11154              :    corresponding dr_vec_info need to be reconnected to the EPILOGUE's
   11155              :    stmt_vec_infos, their statements need to point to their corresponding
   11156              :    copy.  */
   11157              : 
   11158              : static void
   11159         6946 : update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
   11160              : {
   11161         6946 :   loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
   11162         6946 :   hash_map<tree,tree> mapping;
   11163         6946 :   gimple *orig_stmt, *new_stmt;
   11164         6946 :   gimple_stmt_iterator epilogue_gsi;
   11165         6946 :   gphi_iterator epilogue_phi_gsi;
   11166         6946 :   stmt_vec_info stmt_vinfo = NULL, related_vinfo;
   11167         6946 :   basic_block *epilogue_bbs = get_loop_body (epilogue);
   11168         6946 :   unsigned i;
   11169              : 
   11170         6946 :   free (LOOP_VINFO_BBS (epilogue_vinfo));
   11171         6946 :   LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
   11172         6946 :   LOOP_VINFO_NBBS (epilogue_vinfo) = epilogue->num_nodes;
   11173              : 
   11174              :   /* The EPILOGUE loop is a copy of the original loop so they share the same
   11175              :      gimple UIDs.  In this loop we update the loop_vec_info of the EPILOGUE to
   11176              :      point to the copied statements.  */
   11177        20838 :   for (unsigned i = 0; i < epilogue->num_nodes; ++i)
   11178              :     {
   11179        13892 :       for (epilogue_phi_gsi = gsi_start_phis (epilogue_bbs[i]);
   11180        35651 :            !gsi_end_p (epilogue_phi_gsi); gsi_next (&epilogue_phi_gsi))
   11181              :         {
   11182        21759 :           new_stmt = epilogue_phi_gsi.phi ();
   11183              : 
   11184        21759 :           gcc_assert (gimple_uid (new_stmt) > 0);
   11185        21759 :           stmt_vinfo
   11186        21759 :             = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
   11187              : 
   11188        21759 :           STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
   11189              :         }
   11190              : 
   11191        27784 :       for (epilogue_gsi = gsi_start_bb (epilogue_bbs[i]);
   11192       138114 :            !gsi_end_p (epilogue_gsi); gsi_next (&epilogue_gsi))
   11193              :         {
   11194       124222 :           new_stmt = gsi_stmt (epilogue_gsi);
   11195       124222 :           if (is_gimple_debug (new_stmt))
   11196        20049 :             continue;
   11197              : 
   11198       104173 :           gcc_assert (gimple_uid (new_stmt) > 0);
   11199       104173 :           stmt_vinfo
   11200       104173 :             = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
   11201              : 
   11202       104173 :           STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
   11203              : 
   11204       104173 :           related_vinfo = STMT_VINFO_RELATED_STMT (stmt_vinfo);
   11205       104173 :           if (related_vinfo != NULL && related_vinfo != stmt_vinfo)
   11206              :             {
   11207         2197 :               gimple *stmt = STMT_VINFO_STMT (related_vinfo);
   11208              :               /* Set BB such that the assert in
   11209              :                 'get_initial_defs_for_reduction' is able to determine that
   11210              :                 the BB of the related stmt is inside this loop.  */
   11211         2197 :               gimple_set_bb (stmt,
   11212              :                              gimple_bb (new_stmt));
   11213         2197 :               related_vinfo = STMT_VINFO_RELATED_STMT (related_vinfo);
   11214         2197 :               gcc_assert (related_vinfo == NULL
   11215              :                           || related_vinfo == stmt_vinfo);
   11216              :             }
   11217              :         }
   11218              :     }
   11219              : 
   11220         6946 :   struct data_reference *dr;
   11221         6946 :   vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (epilogue_vinfo);
   11222        31380 :   FOR_EACH_VEC_ELT (datarefs, i, dr)
   11223              :     {
   11224        24434 :       orig_stmt = DR_STMT (dr);
   11225        24434 :       gcc_assert (gimple_uid (orig_stmt) > 0);
   11226        24434 :       stmt_vinfo = epilogue_vinfo->stmt_vec_infos[gimple_uid (orig_stmt) - 1];
   11227        24434 :       DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo);
   11228              :     }
   11229              : 
   11230              :   /* Advance data_reference's with the number of iterations of the previous
   11231              :      loop and its prologue.  */
   11232         6946 :   vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
   11233              : 
   11234              :   /* Remember the advancement made.  */
   11235         6946 :   LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
   11236         6946 : }
   11237              : 
   11238              : /*  When vectorizing early break statements instructions that happen before
   11239              :     the early break in the current BB need to be moved to after the early
   11240              :     break.  This function deals with that and assumes that any validity
   11241              :     checks has already been performed.
   11242              : 
   11243              :     While moving the instructions if it encounters a VUSE or VDEF it then
   11244              :     corrects the VUSES as it moves the statements along.  GDEST is the location
   11245              :     in which to insert the new statements.  */
   11246              : 
   11247              : static void
   11248         1486 : move_early_exit_stmts (loop_vec_info loop_vinfo)
   11249              : {
   11250         1486 :   DUMP_VECT_SCOPE ("move_early_exit_stmts");
   11251              : 
   11252         1486 :   if (LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).is_empty ())
   11253         1236 :     return;
   11254              : 
   11255              :   /* Move all stmts that need moving.  */
   11256          250 :   basic_block dest_bb = LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo);
   11257          250 :   gimple_stmt_iterator dest_gsi = gsi_after_labels (dest_bb);
   11258              : 
   11259          250 :   tree last_seen_vuse = NULL_TREE;
   11260          599 :   for (gimple *stmt : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo))
   11261              :     {
   11262              :       /* We have to update crossed degenerate virtual PHIs.  Simply
   11263              :          elide them.  */
   11264          349 :       if (gphi *vphi = dyn_cast <gphi *> (stmt))
   11265              :         {
   11266            7 :           tree vdef = gimple_phi_result (vphi);
   11267            7 :           tree vuse = gimple_phi_arg_def (vphi, 0);
   11268            7 :           imm_use_iterator iter;
   11269            7 :           use_operand_p use_p;
   11270            7 :           gimple *use_stmt;
   11271           23 :           FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
   11272              :             {
   11273           32 :               FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
   11274           16 :                 SET_USE (use_p, vuse);
   11275            7 :             }
   11276            7 :           auto gsi = gsi_for_stmt (stmt);
   11277            7 :           remove_phi_node (&gsi, true);
   11278            7 :           last_seen_vuse = vuse;
   11279            7 :           continue;
   11280            7 :         }
   11281              : 
   11282              :       /* Check to see if statement is still required for vect or has been
   11283              :          elided.  */
   11284          342 :       auto stmt_info = loop_vinfo->lookup_stmt (stmt);
   11285          342 :       if (!stmt_info)
   11286            0 :         continue;
   11287              : 
   11288          342 :       if (dump_enabled_p ())
   11289          177 :         dump_printf_loc (MSG_NOTE, vect_location, "moving stmt %G", stmt);
   11290              : 
   11291          342 :       gimple_stmt_iterator stmt_gsi = gsi_for_stmt (stmt);
   11292          342 :       gsi_move_before (&stmt_gsi, &dest_gsi, GSI_NEW_STMT);
   11293          684 :       last_seen_vuse = gimple_vuse (stmt);
   11294              :     }
   11295              : 
   11296              :   /* Update all the stmts with their new reaching VUSES.  */
   11297          865 :   for (auto p : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
   11298              :     {
   11299          281 :       if (dump_enabled_p ())
   11300          203 :           dump_printf_loc (MSG_NOTE, vect_location,
   11301              :                            "updating vuse to %T for load %G",
   11302              :                            last_seen_vuse, p);
   11303          281 :       gimple_set_vuse (p, last_seen_vuse);
   11304          281 :       update_stmt (p);
   11305              :     }
   11306              : 
   11307              :   /* And update the LC PHIs on exits.  */
   11308         1263 :   for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP  (loop_vinfo)))
   11309          513 :     if (!dominated_by_p (CDI_DOMINATORS, e->src, dest_bb))
   11310          293 :       if (gphi *phi = get_virtual_phi (e->dest))
   11311          543 :         SET_PHI_ARG_DEF_ON_EDGE (phi, e, last_seen_vuse);
   11312              : }
   11313              : 
   11314              : /* Generate adjustment code for early break scalar IVs filling in the value
   11315              :    we created earlier on for LOOP_VINFO_EARLY_BRK_NITERS_VAR.  */
   11316              : 
   11317              : static void
   11318         1486 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
   11319              : {
   11320         1486 :   DUMP_VECT_SCOPE ("vect_update_ivs_after_vectorizer_for_early_breaks");
   11321              : 
   11322         1486 :   if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
   11323              :       /* If no peeling was done then we have no IV to update.  */
   11324         1486 :       || !LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo))
   11325          598 :     return;
   11326              : 
   11327          888 :   tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
   11328          888 :   tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
   11329          888 :   tree ty_var = TREE_TYPE (phi_var);
   11330          888 :   auto loop = LOOP_VINFO_LOOP (loop_vinfo);
   11331          888 :   tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
   11332              : 
   11333              :   /* Remove the existing dummy GIMPLE statement and just keep the def.  */
   11334          888 :   gimple *def = SSA_NAME_DEF_STMT (phi_var);
   11335          888 :   auto def_gsi = gsi_for_stmt (def);
   11336          888 :   gsi_remove (&def_gsi, true);
   11337              : 
   11338          888 :   auto induction_phi = create_phi_node (induc_var, loop->header);
   11339          888 :   tree induc_def = PHI_RESULT (induction_phi);
   11340              : 
   11341              :   /* Create the iv update inside the loop.  */
   11342          888 :   gimple_seq init_stmts = NULL;
   11343          888 :   gimple_seq stmts = NULL;
   11344          888 :   gimple_seq iv_stmts = NULL;
   11345          888 :   tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
   11346              : 
   11347          888 :   tree iter_var;
   11348          888 :   if (POINTER_TYPE_P (ty_var))
   11349            0 :     iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
   11350              :                              tree_iv_incr);
   11351              :   else
   11352              :     {
   11353          888 :       tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
   11354          888 :       iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
   11355              :     }
   11356              : 
   11357          888 :   tree init_var = build_zero_cst (ty_var);
   11358          888 :   if (niters_skip)
   11359            0 :     init_var = gimple_build (&init_stmts, MINUS_EXPR, ty_var, init_var,
   11360              :                              gimple_convert (&init_stmts, ty_var, niters_skip));
   11361              : 
   11362          888 :   add_phi_arg (induction_phi, iter_var,
   11363              :                loop_latch_edge (loop), UNKNOWN_LOCATION);
   11364          888 :   add_phi_arg (induction_phi, init_var,
   11365              :                loop_preheader_edge (loop), UNKNOWN_LOCATION);
   11366              : 
   11367              :   /* Find the first insertion point in the BB.  */
   11368          888 :   auto pe = loop_preheader_edge (loop);
   11369              : 
   11370              :   /* If we've done any peeling, calculate the peeling adjustment needed to the
   11371              :      final IV.  */
   11372          888 :   if (niters_skip)
   11373              :     {
   11374            0 :       tree induc_type = TREE_TYPE (induc_def);
   11375            0 :       tree s_induc_type = signed_type_for (induc_type);
   11376            0 :       induc_def = gimple_build (&iv_stmts, MAX_EXPR, s_induc_type,
   11377              :                                 gimple_convert (&iv_stmts, s_induc_type,
   11378              :                                                 induc_def),
   11379              :                                 build_zero_cst (s_induc_type));
   11380            0 :       auto stmt = gimple_build_assign (phi_var,
   11381              :                                        gimple_convert (&iv_stmts, induc_type,
   11382              :                                                        induc_def));
   11383            0 :       gimple_seq_add_stmt_without_update (&iv_stmts, stmt);
   11384            0 :       basic_block exit_bb = NULL;
   11385              :       /* Identify the early exit merge block.  I wish we had stored this.  */
   11386            0 :       for (auto e : get_loop_exit_edges (loop))
   11387            0 :         if (e != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
   11388              :           {
   11389            0 :             exit_bb = e->dest;
   11390            0 :             break;
   11391            0 :           }
   11392              : 
   11393            0 :       gcc_assert (exit_bb);
   11394            0 :       auto exit_gsi = gsi_after_labels (exit_bb);
   11395            0 :       gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
   11396              :     }
   11397              :   /* Write the init_stmts in the loop-preheader block.  */
   11398          888 :   auto psi = gsi_last_nondebug_bb (pe->src);
   11399          888 :   gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
   11400              : 
   11401              :   /* Write the adjustments at the end of the iv increment.  */
   11402          888 :   bool insert_after;
   11403          888 :   gimple_stmt_iterator incr_gsi;
   11404          888 :   standard_iv_increment_position (loop, &incr_gsi, &insert_after);
   11405              : 
   11406          888 :   if (insert_after)
   11407            0 :     gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT);
   11408              :   else
   11409          888 :     gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT);
   11410              : }
   11411              : 
   11412              : /* Function vect_transform_loop.
   11413              : 
   11414              :    The analysis phase has determined that the loop is vectorizable.
   11415              :    Vectorize the loop - created vectorized stmts to replace the scalar
   11416              :    stmts in the loop, and update the loop exit condition.
   11417              :    Returns scalar epilogue loop if any.  */
   11418              : 
   11419              : class loop *
   11420        64392 : vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
   11421              : {
   11422        64392 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   11423        64392 :   class loop *epilogue = NULL;
   11424        64392 :   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
   11425        64392 :   int nbbs = loop->num_nodes;
   11426        64392 :   int i;
   11427        64392 :   tree niters_vector = NULL_TREE;
   11428        64392 :   tree step_vector = NULL_TREE;
   11429        64392 :   tree niters_vector_mult_vf = NULL_TREE;
   11430        64392 :   poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   11431        64392 :   unsigned int lowest_vf = constant_lower_bound (vf);
   11432        64392 :   gimple *stmt;
   11433        64392 :   bool check_profitability = false;
   11434        64392 :   unsigned int th;
   11435        64392 :   bool flat = maybe_flat_loop_profile (loop);
   11436        64392 :   bool uncounted_p = LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo);
   11437              : 
   11438        64392 :   DUMP_VECT_SCOPE ("vec_transform_loop");
   11439              : 
   11440        64392 :   if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
   11441        57446 :     loop_vinfo->shared->check_datarefs ();
   11442              : 
   11443              :   /* Use the more conservative vectorization threshold.  If the number
   11444              :      of iterations is constant assume the cost check has been performed
   11445              :      by our caller.  If the threshold makes all loops profitable that
   11446              :      run at least the (estimated) vectorization factor number of times
   11447              :      checking is pointless, too.  */
   11448        64392 :   th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
   11449        64392 :   if (vect_apply_runtime_profitability_check_p (loop_vinfo))
   11450              :     {
   11451        19065 :       if (dump_enabled_p ())
   11452          179 :         dump_printf_loc (MSG_NOTE, vect_location,
   11453              :                          "Profitability threshold is %d loop iterations.\n",
   11454              :                          th);
   11455              :       check_profitability = true;
   11456              :     }
   11457              : 
   11458              :   /* Make sure there exists a single-predecessor exit bb.  Do this before
   11459              :      versioning.   */
   11460        64392 :   edge e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
   11461        64392 :   if (! single_pred_p (e->dest) && !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
   11462              :     {
   11463        19517 :       split_loop_exit_edge (e, true);
   11464        19517 :       if (dump_enabled_p ())
   11465         2419 :         dump_printf (MSG_NOTE, "split exit edge\n");
   11466              :     }
   11467              : 
   11468              :   /* Version the loop first, if required, so the profitability check
   11469              :      comes first.  */
   11470              : 
   11471        64392 :   if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
   11472              :     {
   11473         3906 :       class loop *sloop
   11474         3906 :         = vect_loop_versioning (loop_vinfo, loop_vectorized_call);
   11475         3906 :       sloop->force_vectorize = false;
   11476         3906 :       check_profitability = false;
   11477              :     }
   11478              : 
   11479              :   /* Make sure there exists a single-predecessor exit bb also on the
   11480              :      scalar loop copy.  Do this after versioning but before peeling
   11481              :      so CFG structure is fine for both scalar and if-converted loop
   11482              :      to make slpeel_duplicate_current_defs_from_edges face matched
   11483              :      loop closed PHI nodes on the exit.  */
   11484        64392 :   if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
   11485              :     {
   11486         7307 :       e = LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo);
   11487         7307 :       if (! single_pred_p (e->dest))
   11488              :         {
   11489         7009 :           split_loop_exit_edge (e, true);
   11490         7009 :           if (dump_enabled_p ())
   11491         1209 :             dump_printf (MSG_NOTE, "split exit edge of scalar loop\n");
   11492              :         }
   11493              :     }
   11494              : 
   11495        64392 :   tree niters = vect_build_loop_niters (loop_vinfo);
   11496        64392 :   LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = niters;
   11497        64392 :   tree nitersm1 = unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo));
   11498        64392 :   tree advance;
   11499        64392 :   drs_init_vec orig_drs_init;
   11500        64392 :   bool niters_no_overflow = uncounted_p ? false /* Not known.  */
   11501        64347 :                                         : loop_niters_no_overflow (loop_vinfo);
   11502              : 
   11503        64392 :   epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
   11504              :                               &step_vector, &niters_vector_mult_vf, th,
   11505              :                               check_profitability, niters_no_overflow,
   11506              :                               &advance);
   11507              : 
   11508        64392 :   LOOP_VINFO_IV_INCREMENT (loop_vinfo)
   11509        64392 :     = vect_get_loop_iv_increment (loop_vinfo);
   11510              : 
   11511              :   /* Assign hierarchical discriminators to the vectorized loop.  */
   11512        64392 :   poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   11513        64392 :   unsigned int vf_int = constant_lower_bound (vf_val);
   11514        64392 :   if (vf_int > DISCR_MULTIPLICITY_MAX)
   11515              :     vf_int = DISCR_MULTIPLICITY_MAX;
   11516              : 
   11517              :   /* Assign unique copy_id dynamically instead of using hardcoded constants.
   11518              :      Epilogue and main vectorized loops get different copy_ids.  */
   11519        64392 :   gimple *loop_last = last_nondebug_stmt (loop->header);
   11520        64392 :   location_t loop_loc
   11521        64392 :     = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION;
   11522        64118 :   if (loop_loc != UNKNOWN_LOCATION)
   11523              :     {
   11524        53515 :       unsigned int copyid = allocate_copyid_base (loop_loc, 1);
   11525        53515 :       assign_discriminators_to_loop (loop, vf_int, copyid);
   11526              :     }
   11527        64392 :   if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
   11528        64392 :       && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
   11529              :     {
   11530              :       /* Ifcvt duplicates loop preheader, loop body and produces an basic
   11531              :          block after loop exit.  We need to scale all that.  */
   11532           90 :       basic_block preheader
   11533           90 :         = loop_preheader_edge (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))->src;
   11534           90 :       preheader->count
   11535              :         = preheader->count.apply_probability
   11536           90 :               (LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
   11537           90 :       scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo),
   11538              :                               LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
   11539           90 :       LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo)->dest->count = preheader->count;
   11540              :     }
   11541              : 
   11542        64392 :   if (niters_vector == NULL_TREE && !uncounted_p)
   11543              :     {
   11544        29360 :       if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
   11545        29360 :           && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
   11546        59532 :           && known_eq (lowest_vf, vf))
   11547              :         {
   11548        29355 :           niters_vector
   11549        29355 :             = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
   11550        29355 :                              LOOP_VINFO_INT_NITERS (loop_vinfo) / lowest_vf);
   11551        29355 :           step_vector = build_one_cst (TREE_TYPE (niters));
   11552              :         }
   11553          822 :       else if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
   11554            2 :         vect_gen_vector_loop_niters (loop_vinfo, niters, &niters_vector,
   11555              :                                      &step_vector, niters_no_overflow);
   11556              :       else
   11557              :         /* vect_do_peeling subtracted the number of peeled prologue
   11558              :            iterations from LOOP_VINFO_NITERS.  */
   11559          820 :         vect_gen_vector_loop_niters (loop_vinfo, LOOP_VINFO_NITERS (loop_vinfo),
   11560              :                                      &niters_vector, &step_vector,
   11561              :                                      niters_no_overflow);
   11562              :     }
   11563              : 
   11564              :   /* 1) Make sure the loop header has exactly two entries
   11565              :      2) Make sure we have a preheader basic block.  */
   11566              : 
   11567        64392 :   gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
   11568              : 
   11569        64392 :   split_edge (loop_preheader_edge (loop));
   11570              : 
   11571        64392 :   if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
   11572              :     /* This will deal with any possible peeling.  */
   11573            2 :     vect_prepare_for_masked_peels (loop_vinfo);
   11574              : 
   11575              :   /* Handle any code motion that we need to for early-break vectorization after
   11576              :      we've done peeling but just before we start vectorizing.  */
   11577        64392 :   if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
   11578              :     {
   11579         1486 :       vect_update_ivs_after_vectorizer_for_early_breaks (loop_vinfo);
   11580         1486 :       move_early_exit_stmts (loop_vinfo);
   11581              :     }
   11582              : 
   11583              :   /* Remove existing clobber stmts and prefetches.  */
   11584       196473 :   for (i = 0; i < nbbs; i++)
   11585              :     {
   11586       132081 :       basic_block bb = bbs[i];
   11587      1138191 :       for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);)
   11588              :         {
   11589       874029 :           stmt = gsi_stmt (si);
   11590       874029 :           if (gimple_clobber_p (stmt)
   11591       874029 :               || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH))
   11592              :             {
   11593          102 :               unlink_stmt_vdef (stmt);
   11594          102 :               gsi_remove (&si, true);
   11595          102 :               release_defs (stmt);
   11596              :             }
   11597              :           else
   11598       873927 :             gsi_next (&si);
   11599              :         }
   11600              :     }
   11601              : 
   11602              :   /* Schedule the SLP instances.  */
   11603        64392 :   if (!loop_vinfo->slp_instances.is_empty ())
   11604              :     {
   11605        64392 :       DUMP_VECT_SCOPE ("scheduling SLP instances");
   11606        64392 :       vect_schedule_slp (loop_vinfo, LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
   11607              :                          false);
   11608              :     }
   11609              : 
   11610              :   /* Generate the loop invariant statements.  */
   11611        64392 :   if (!gimple_seq_empty_p (LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo)))
   11612              :     {
   11613           80 :       if (dump_enabled_p ())
   11614           26 :         dump_printf_loc (MSG_NOTE, vect_location,
   11615              :                          "------>generating loop invariant statements\n");
   11616           80 :       gimple_stmt_iterator gsi;
   11617           80 :       gsi = gsi_after_labels (loop_preheader_edge (loop)->src);
   11618           80 :       gsi_insert_seq_before (&gsi, LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo),
   11619              :                              GSI_CONTINUE_LINKING);
   11620              :     }
   11621              : 
   11622              :   /* Stub out scalar statements that must not survive vectorization and
   11623              :      were not picked as relevant in any SLP instance.
   11624              :      Doing this here helps with grouped statements, or statements that
   11625              :      are involved in patterns.  */
   11626       196473 :   for (i = 0; i < nbbs; i++)
   11627              :     {
   11628       132081 :       basic_block bb = bbs[i];
   11629       132081 :       stmt_vec_info stmt_info;
   11630       264162 :       for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
   11631      1733656 :            !gsi_end_p (gsi); gsi_next (&gsi))
   11632              :         {
   11633      1601575 :           gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
   11634         6746 :           if (!call || !gimple_call_internal_p (call))
   11635      1596198 :             continue;
   11636         5377 :           internal_fn ifn = gimple_call_internal_fn (call);
   11637         5377 :           if (ifn == IFN_MASK_LOAD)
   11638              :             {
   11639          699 :               tree lhs = gimple_get_lhs (call);
   11640          699 :               if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
   11641              :                 {
   11642            0 :                   tree zero = build_zero_cst (TREE_TYPE (lhs));
   11643            0 :                   gimple *new_stmt = gimple_build_assign (lhs, zero);
   11644            0 :                   gsi_replace (&gsi, new_stmt, true);
   11645              :                 }
   11646              :             }
   11647         4678 :           else if (conditional_internal_fn_code (ifn) != ERROR_MARK)
   11648              :             {
   11649         2403 :               tree lhs = gimple_get_lhs (call);
   11650         2403 :               if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
   11651              :                 {
   11652            0 :                   tree else_arg
   11653            0 :                     = gimple_call_arg (call, gimple_call_num_args (call) - 1);
   11654            0 :                   gimple *new_stmt = gimple_build_assign (lhs, else_arg);
   11655            0 :                   gsi_replace (&gsi, new_stmt, true);
   11656              :                 }
   11657              :             }
   11658         2275 :           else if (ifn == IFN_MASK_CALL
   11659           14 :                    && (stmt_info = loop_vinfo->lookup_stmt (call))
   11660           14 :                    && !STMT_VINFO_RELEVANT_P (stmt_info)
   11661         2289 :                    && !STMT_VINFO_LIVE_P (stmt_info))
   11662              :             {
   11663              :               /* With -fno-tree-{dce,dse} we can end up with dead
   11664              :                  .MASK_CALL without side-effects and thus with a (dead)
   11665              :                  LHS.  So we cannot assert there is no LHS.  */
   11666           14 :               loop_vinfo->remove_stmt (stmt_info);
   11667              :             }
   11668              :         }
   11669              :     }
   11670              : 
   11671        64392 :   if (!uncounted_p)
   11672              :     {
   11673              :       /* The vectorization factor is always > 1, so if we use an IV increment of
   11674              :          1.  A zero NITERS becomes a nonzero NITERS_VECTOR.  */
   11675        64347 :       if (integer_onep (step_vector))
   11676        64323 :         niters_no_overflow = true;
   11677              : 
   11678        64347 :       vect_set_loop_condition (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
   11679              :                                loop_vinfo, niters_vector, step_vector,
   11680              :                                niters_vector_mult_vf, !niters_no_overflow);
   11681              :     }
   11682              : 
   11683        64392 :   unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
   11684              : 
   11685              :   /* True if the final iteration might not handle a full vector's
   11686              :      worth of scalar iterations.  */
   11687       128784 :   bool final_iter_may_be_partial
   11688        64392 :     = LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
   11689        64392 :       || LOOP_VINFO_EARLY_BREAKS (loop_vinfo);
   11690              : 
   11691              :   /* +1 to convert latch counts to loop iteration counts.  */
   11692        64392 :   int bias_for_lowest = 1;
   11693              : 
   11694              :   /* When we are peeling for gaps then we take away one scalar iteration
   11695              :      from the vector loop.  Thus we can adjust the upper bound by one
   11696              :      scalar iteration.  But only when we know the bound applies to the
   11697              :      IV exit test which might not be true when we have multiple exits.  */
   11698        64392 :   if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
   11699       125406 :     bias_for_lowest -= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
   11700              : 
   11701        64392 :   int bias_for_assumed = bias_for_lowest;
   11702        64392 :   int alignment_npeels = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
   11703        64392 :   if (alignment_npeels && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
   11704              :     {
   11705              :       /* When the amount of peeling is known at compile time, the first
   11706              :          iteration will have exactly alignment_npeels active elements.
   11707              :          In the worst case it will have at least one.  */
   11708            2 :       int min_first_active = (alignment_npeels > 0 ? alignment_npeels : 1);
   11709            2 :       bias_for_lowest += lowest_vf - min_first_active;
   11710            2 :       bias_for_assumed += assumed_vf - min_first_active;
   11711              :     }
   11712              :   /* In these calculations the "- 1" converts loop iteration counts
   11713              :      back to latch counts.  */
   11714        64392 :   if (loop->any_upper_bound)
   11715              :     {
   11716        64374 :       loop_vec_info main_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
   11717        64374 :       loop->nb_iterations_upper_bound
   11718        64374 :         = (final_iter_may_be_partial
   11719        65864 :            ? wi::udiv_ceil (loop->nb_iterations_upper_bound + bias_for_lowest,
   11720         2980 :                             lowest_vf) - 1
   11721        62884 :            : wi::udiv_floor (loop->nb_iterations_upper_bound + bias_for_lowest,
   11722       125768 :                              lowest_vf) - 1);
   11723        64374 :       if (main_vinfo
   11724              :           /* Both peeling for alignment and peeling for gaps can end up
   11725              :              with the scalar epilogue running for more than VF-1 iterations.  */
   11726         6946 :           && !main_vinfo->peeling_for_alignment
   11727         6899 :           && !main_vinfo->peeling_for_gaps)
   11728              :         {
   11729         6719 :           unsigned int bound;
   11730         6719 :           poly_uint64 main_iters
   11731         6719 :             = upper_bound (LOOP_VINFO_VECT_FACTOR (main_vinfo),
   11732              :                            LOOP_VINFO_COST_MODEL_THRESHOLD (main_vinfo));
   11733         6719 :           main_iters
   11734         6719 :             = upper_bound (main_iters,
   11735         6719 :                            LOOP_VINFO_VERSIONING_THRESHOLD (main_vinfo));
   11736         6719 :           if (can_div_away_from_zero_p (main_iters,
   11737         6719 :                                         LOOP_VINFO_VECT_FACTOR (loop_vinfo),
   11738              :                                         &bound))
   11739         6719 :             loop->nb_iterations_upper_bound
   11740         6719 :               = wi::umin ((bound_wide_int) (bound - 1),
   11741         6719 :                           loop->nb_iterations_upper_bound);
   11742              :       }
   11743              :   }
   11744        64392 :   if (loop->any_likely_upper_bound)
   11745        64374 :     loop->nb_iterations_likely_upper_bound
   11746        64374 :       = (final_iter_may_be_partial
   11747        65864 :          ? wi::udiv_ceil (loop->nb_iterations_likely_upper_bound
   11748         1490 :                           + bias_for_lowest, lowest_vf) - 1
   11749        62884 :          : wi::udiv_floor (loop->nb_iterations_likely_upper_bound
   11750        64374 :                            + bias_for_lowest, lowest_vf) - 1);
   11751        64392 :   if (loop->any_estimate)
   11752        37566 :     loop->nb_iterations_estimate
   11753        37566 :       = (final_iter_may_be_partial
   11754        38269 :          ? wi::udiv_ceil (loop->nb_iterations_estimate + bias_for_assumed,
   11755         1406 :                           assumed_vf) - 1
   11756        36863 :          : wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
   11757        74429 :                            assumed_vf) - 1);
   11758        64392 :   scale_profile_for_vect_loop (loop_vinfo, loop,
   11759              :                                LOOP_VINFO_MAIN_EXIT (loop_vinfo), assumed_vf,
   11760              :                                flat);
   11761              : 
   11762        64392 :   if (dump_enabled_p ())
   11763              :     {
   11764        11496 :       if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
   11765              :         {
   11766         9935 :           dump_printf_loc (MSG_NOTE, vect_location,
   11767              :                            "LOOP VECTORIZED\n");
   11768         9935 :           if (loop->inner)
   11769          344 :             dump_printf_loc (MSG_NOTE, vect_location,
   11770              :                              "OUTER LOOP VECTORIZED\n");
   11771         9935 :           dump_printf (MSG_NOTE, "\n");
   11772              :         }
   11773              :       else
   11774         1561 :         dump_printf_loc (MSG_NOTE, vect_location,
   11775              :                          "LOOP EPILOGUE VECTORIZED (MODE=%s)\n",
   11776         1561 :                          GET_MODE_NAME (loop_vinfo->vector_mode));
   11777              :     }
   11778              : 
   11779              :   /* Loops vectorized with a variable factor won't benefit from
   11780              :      unrolling/peeling.  */
   11781        64392 :   if (!vf.is_constant ())
   11782              :     {
   11783              :       loop->unroll = 1;
   11784              :       if (dump_enabled_p ())
   11785              :         dump_printf_loc (MSG_NOTE, vect_location, "Disabling unrolling due to"
   11786              :                          " variable-length vectorization factor\n");
   11787              :     }
   11788              : 
   11789              :   /* When we have unrolled the loop due to a user requested value we should
   11790              :      leave it up to the RTL unroll heuristics to determine if it's still worth
   11791              :      while to unroll more.  */
   11792        64392 :   if (LOOP_VINFO_USER_UNROLL (loop_vinfo))
   11793           44 :     loop->unroll = 0;
   11794              : 
   11795              :   /* Free SLP instances here because otherwise stmt reference counting
   11796              :      won't work.  */
   11797        64392 :   slp_instance instance;
   11798       157263 :   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
   11799        92871 :     vect_free_slp_instance (instance);
   11800        64392 :   LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
   11801              :   /* Clear-up safelen field since its value is invalid after vectorization
   11802              :      since vectorized loop can have loop-carried dependencies.  */
   11803        64392 :   loop->safelen = 0;
   11804              : 
   11805        64392 :   if (epilogue)
   11806              :     {
   11807              :       /* Accumulate past advancements made.  */
   11808         6946 :       if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
   11809           91 :         advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
   11810              :                                LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
   11811              :                                advance);
   11812         6946 :       update_epilogue_loop_vinfo (epilogue, advance);
   11813              : 
   11814         6946 :       epilogue->simduid = loop->simduid;
   11815         6946 :       epilogue->force_vectorize = loop->force_vectorize;
   11816         6946 :       epilogue->dont_vectorize = false;
   11817              :     }
   11818              : 
   11819        64392 :   return epilogue;
   11820        64392 : }
   11821              : 
   11822              : /* The code below is trying to perform simple optimization - revert
   11823              :    if-conversion for masked stores, i.e. if the mask of a store is zero
   11824              :    do not perform it and all stored value producers also if possible.
   11825              :    For example,
   11826              :      for (i=0; i<n; i++)
   11827              :        if (c[i])
   11828              :         {
   11829              :           p1[i] += 1;
   11830              :           p2[i] = p3[i] +2;
   11831              :         }
   11832              :    this transformation will produce the following semi-hammock:
   11833              : 
   11834              :    if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
   11835              :      {
   11836              :        vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
   11837              :        vect__12.22_172 = vect__11.19_170 + vect_cst__171;
   11838              :        MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
   11839              :        vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
   11840              :        vect__19.28_184 = vect__18.25_182 + vect_cst__183;
   11841              :        MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
   11842              :      }
   11843              : */
   11844              : 
   11845              : void
   11846          499 : optimize_mask_stores (class loop *loop)
   11847              : {
   11848          499 :   basic_block *bbs = get_loop_body (loop);
   11849          499 :   unsigned nbbs = loop->num_nodes;
   11850          499 :   unsigned i;
   11851          499 :   basic_block bb;
   11852          499 :   class loop *bb_loop;
   11853          499 :   gimple_stmt_iterator gsi;
   11854          499 :   gimple *stmt;
   11855          499 :   auto_vec<gimple *> worklist;
   11856          499 :   auto_purge_vect_location sentinel;
   11857              : 
   11858          499 :   vect_location = find_loop_location (loop);
   11859              :   /* Pick up all masked stores in loop if any.  */
   11860         1996 :   for (i = 0; i < nbbs; i++)
   11861              :     {
   11862          998 :       bb = bbs[i];
   11863        16957 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
   11864        14961 :            gsi_next (&gsi))
   11865              :         {
   11866        14961 :           stmt = gsi_stmt (gsi);
   11867        14961 :           if (gimple_call_internal_p (stmt, IFN_MASK_STORE))
   11868          673 :             worklist.safe_push (stmt);
   11869              :         }
   11870              :     }
   11871              : 
   11872          499 :   free (bbs);
   11873          499 :   if (worklist.is_empty ())
   11874           68 :     return;
   11875              : 
   11876              :   /* Loop has masked stores.  */
   11877         1087 :   while (!worklist.is_empty ())
   11878              :     {
   11879          656 :       gimple *last, *last_store;
   11880          656 :       edge e, efalse;
   11881          656 :       tree mask;
   11882          656 :       basic_block store_bb, join_bb;
   11883          656 :       gimple_stmt_iterator gsi_to;
   11884          656 :       tree vdef, new_vdef;
   11885          656 :       gphi *phi;
   11886          656 :       tree vectype;
   11887          656 :       tree zero;
   11888              : 
   11889          656 :       last = worklist.pop ();
   11890          656 :       mask = gimple_call_arg (last, 2);
   11891          656 :       bb = gimple_bb (last);
   11892              :       /* Create then_bb and if-then structure in CFG, then_bb belongs to
   11893              :          the same loop as if_bb.  It could be different to LOOP when two
   11894              :          level loop-nest is vectorized and mask_store belongs to the inner
   11895              :          one.  */
   11896          656 :       e = split_block (bb, last);
   11897          656 :       bb_loop = bb->loop_father;
   11898          656 :       gcc_assert (loop == bb_loop || flow_loop_nested_p (loop, bb_loop));
   11899          656 :       join_bb = e->dest;
   11900          656 :       store_bb = create_empty_bb (bb);
   11901          656 :       add_bb_to_loop (store_bb, bb_loop);
   11902          656 :       e->flags = EDGE_TRUE_VALUE;
   11903          656 :       efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
   11904              :       /* Put STORE_BB to likely part.  */
   11905          656 :       efalse->probability = profile_probability::likely ();
   11906          656 :       e->probability = efalse->probability.invert ();
   11907          656 :       store_bb->count = efalse->count ();
   11908          656 :       make_single_succ_edge (store_bb, join_bb, EDGE_FALLTHRU);
   11909          656 :       if (dom_info_available_p (CDI_DOMINATORS))
   11910          656 :         set_immediate_dominator (CDI_DOMINATORS, store_bb, bb);
   11911          656 :       if (dump_enabled_p ())
   11912          328 :         dump_printf_loc (MSG_NOTE, vect_location,
   11913              :                          "Create new block %d to sink mask stores.",
   11914              :                          store_bb->index);
   11915              :       /* Create vector comparison with boolean result.  */
   11916          656 :       vectype = TREE_TYPE (mask);
   11917          656 :       zero = build_zero_cst (vectype);
   11918          656 :       stmt = gimple_build_cond (EQ_EXPR, mask, zero, NULL_TREE, NULL_TREE);
   11919          656 :       gsi = gsi_last_bb (bb);
   11920          656 :       gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
   11921              :       /* Create new PHI node for vdef of the last masked store:
   11922              :          .MEM_2 = VDEF <.MEM_1>
   11923              :          will be converted to
   11924              :          .MEM.3 = VDEF <.MEM_1>
   11925              :          and new PHI node will be created in join bb
   11926              :          .MEM_2 = PHI <.MEM_1, .MEM_3>
   11927              :       */
   11928          656 :       vdef = gimple_vdef (last);
   11929          656 :       new_vdef = make_ssa_name (gimple_vop (cfun), last);
   11930          656 :       gimple_set_vdef (last, new_vdef);
   11931          656 :       phi = create_phi_node (vdef, join_bb);
   11932          656 :       add_phi_arg (phi, new_vdef, EDGE_SUCC (store_bb, 0), UNKNOWN_LOCATION);
   11933              : 
   11934              :       /* Put all masked stores with the same mask to STORE_BB if possible.  */
   11935          690 :       while (true)
   11936              :         {
   11937          673 :           gimple_stmt_iterator gsi_from;
   11938          673 :           gimple *stmt1 = NULL;
   11939              : 
   11940              :           /* Move masked store to STORE_BB.  */
   11941          673 :           last_store = last;
   11942          673 :           gsi = gsi_for_stmt (last);
   11943          673 :           gsi_from = gsi;
   11944              :           /* Shift GSI to the previous stmt for further traversal.  */
   11945          673 :           gsi_prev (&gsi);
   11946          673 :           gsi_to = gsi_start_bb (store_bb);
   11947          673 :           gsi_move_before (&gsi_from, &gsi_to);
   11948              :           /* Setup GSI_TO to the non-empty block start.  */
   11949          673 :           gsi_to = gsi_start_bb (store_bb);
   11950          673 :           if (dump_enabled_p ())
   11951          344 :             dump_printf_loc (MSG_NOTE, vect_location,
   11952              :                              "Move stmt to created bb\n%G", last);
   11953              :           /* Move all stored value producers if possible.  */
   11954         5049 :           while (!gsi_end_p (gsi))
   11955              :             {
   11956         5048 :               tree lhs;
   11957         5048 :               imm_use_iterator imm_iter;
   11958         5048 :               use_operand_p use_p;
   11959         5048 :               bool res;
   11960              : 
   11961              :               /* Skip debug statements.  */
   11962         5048 :               if (is_gimple_debug (gsi_stmt (gsi)))
   11963              :                 {
   11964            7 :                   gsi_prev (&gsi);
   11965         3156 :                   continue;
   11966              :                 }
   11967         5041 :               stmt1 = gsi_stmt (gsi);
   11968              :               /* Do not consider statements writing to memory or having
   11969              :                  volatile operand.  */
   11970         9905 :               if (gimple_vdef (stmt1)
   11971         9905 :                   || gimple_has_volatile_ops (stmt1))
   11972              :                 break;
   11973         4864 :               gsi_from = gsi;
   11974         4864 :               gsi_prev (&gsi);
   11975         4864 :               lhs = gimple_get_lhs (stmt1);
   11976         4864 :               if (!lhs)
   11977              :                 break;
   11978              : 
   11979              :               /* LHS of vectorized stmt must be SSA_NAME.  */
   11980         4864 :               if (TREE_CODE (lhs) != SSA_NAME)
   11981              :                 break;
   11982              : 
   11983         4864 :               if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
   11984              :                 {
   11985              :                   /* Remove dead scalar statement.  */
   11986         3466 :                   if (has_zero_uses (lhs))
   11987              :                     {
   11988         3149 :                       gsi_remove (&gsi_from, true);
   11989         3149 :                       release_defs (stmt1);
   11990         3149 :                       continue;
   11991              :                     }
   11992              :                 }
   11993              : 
   11994              :               /* Check that LHS does not have uses outside of STORE_BB.  */
   11995         1715 :               res = true;
   11996         2959 :               FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
   11997              :                 {
   11998         1739 :                   gimple *use_stmt;
   11999         1739 :                   use_stmt = USE_STMT (use_p);
   12000         1739 :                   if (is_gimple_debug (use_stmt))
   12001            0 :                     continue;
   12002         1739 :                   if (gimple_bb (use_stmt) != store_bb)
   12003              :                     {
   12004              :                       res = false;
   12005              :                       break;
   12006              :                     }
   12007         1715 :                 }
   12008         1715 :               if (!res)
   12009              :                 break;
   12010              : 
   12011         1220 :               if (gimple_vuse (stmt1)
   12012         1683 :                   && gimple_vuse (stmt1) != gimple_vuse (last_store))
   12013              :                 break;
   12014              : 
   12015              :               /* Can move STMT1 to STORE_BB.  */
   12016         1220 :               if (dump_enabled_p ())
   12017          620 :                 dump_printf_loc (MSG_NOTE, vect_location,
   12018              :                                  "Move stmt to created bb\n%G", stmt1);
   12019         1220 :               gsi_move_before (&gsi_from, &gsi_to);
   12020              :               /* Shift GSI_TO for further insertion.  */
   12021         1220 :               gsi_prev (&gsi_to);
   12022              :             }
   12023              :           /* Put other masked stores with the same mask to STORE_BB.  */
   12024          673 :           if (worklist.is_empty ()
   12025          242 :               || gimple_call_arg (worklist.last (), 2) != mask
   12026           17 :               || worklist.last () != stmt1)
   12027              :             break;
   12028           17 :           last = worklist.pop ();
   12029           17 :         }
   12030         1312 :       add_phi_arg (phi, gimple_vuse (last_store), e, UNKNOWN_LOCATION);
   12031              :     }
   12032          499 : }
   12033              : 
   12034              : /* Decide whether it is possible to use a zero-based induction variable
   12035              :    when vectorizing LOOP_VINFO with partial vectors.  If it is, return
   12036              :    the value that the induction variable must be able to hold in order
   12037              :    to ensure that the rgroups eventually have no active vector elements.
   12038              :    Return -1 otherwise.  */
   12039              : 
   12040              : widest_int
   12041        48264 : vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo)
   12042              : {
   12043        48264 :   tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
   12044        48264 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   12045        48264 :   unsigned HOST_WIDE_INT max_vf = vect_max_vf (loop_vinfo);
   12046              : 
   12047              :   /* Calculate the value that the induction variable must be able
   12048              :      to hit in order to ensure that we end the loop with an all-false mask.
   12049              :      This involves adding the maximum number of inactive trailing scalar
   12050              :      iterations.  */
   12051        48264 :   widest_int iv_limit = -1;
   12052        48264 :   if (max_loop_iterations (loop, &iv_limit))
   12053              :     {
   12054        48264 :       if (niters_skip)
   12055              :         {
   12056              :           /* Add the maximum number of skipped iterations to the
   12057              :              maximum iteration count.  */
   12058            0 :           if (TREE_CODE (niters_skip) == INTEGER_CST)
   12059            0 :             iv_limit += wi::to_widest (niters_skip);
   12060              :           else
   12061            0 :             iv_limit += max_vf - 1;
   12062              :         }
   12063        48264 :       else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
   12064              :         /* Make a conservatively-correct assumption.  */
   12065          326 :         iv_limit += max_vf - 1;
   12066              : 
   12067              :       /* IV_LIMIT is the maximum number of latch iterations, which is also
   12068              :          the maximum in-range IV value.  Round this value down to the previous
   12069              :          vector alignment boundary and then add an extra full iteration.  */
   12070        48264 :       poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   12071        48264 :       iv_limit = (iv_limit & -(int) known_alignment (vf)) + max_vf;
   12072              :     }
   12073        48264 :   return iv_limit;
   12074              : }
   12075              : 
   12076              : /* For the given rgroup_controls RGC, check whether an induction variable
   12077              :    would ever hit a value that produces a set of all-false masks or zero
   12078              :    lengths before wrapping around.  Return true if it's possible to wrap
   12079              :    around before hitting the desirable value, otherwise return false.  */
   12080              : 
   12081              : bool
   12082            0 : vect_rgroup_iv_might_wrap_p (loop_vec_info loop_vinfo, rgroup_controls *rgc)
   12083              : {
   12084            0 :   widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
   12085              : 
   12086            0 :   if (iv_limit == -1)
   12087              :     return true;
   12088              : 
   12089            0 :   tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
   12090            0 :   unsigned int compare_precision = TYPE_PRECISION (compare_type);
   12091            0 :   unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
   12092              : 
   12093            0 :   if (wi::min_precision (iv_limit * nitems, UNSIGNED) > compare_precision)
   12094            0 :     return true;
   12095              : 
   12096              :   return false;
   12097            0 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.