Line data Source code
1 : /* Loop Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 : Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #define INCLUDE_ALGORITHM
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "cfghooks.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "optabs-tree.h"
35 : #include "memmodel.h"
36 : #include "optabs.h"
37 : #include "diagnostic-core.h"
38 : #include "fold-const.h"
39 : #include "stor-layout.h"
40 : #include "cfganal.h"
41 : #include "gimplify.h"
42 : #include "gimple-iterator.h"
43 : #include "gimplify-me.h"
44 : #include "tree-ssa-loop-ivopts.h"
45 : #include "tree-ssa-loop-manip.h"
46 : #include "tree-ssa-loop-niter.h"
47 : #include "tree-ssa-loop.h"
48 : #include "cfgloop.h"
49 : #include "tree-scalar-evolution.h"
50 : #include "tree-vectorizer.h"
51 : #include "gimple-fold.h"
52 : #include "cgraph.h"
53 : #include "tree-cfg.h"
54 : #include "tree-if-conv.h"
55 : #include "internal-fn.h"
56 : #include "tree-vector-builder.h"
57 : #include "vec-perm-indices.h"
58 : #include "tree-eh.h"
59 : #include "case-cfn-macros.h"
60 : #include "langhooks.h"
61 : #include "opts.h"
62 : #include "hierarchical_discriminator.h"
63 :
64 : /* Loop Vectorization Pass.
65 :
66 : This pass tries to vectorize loops.
67 :
68 : For example, the vectorizer transforms the following simple loop:
69 :
70 : short a[N]; short b[N]; short c[N]; int i;
71 :
72 : for (i=0; i<N; i++){
73 : a[i] = b[i] + c[i];
74 : }
75 :
76 : as if it was manually vectorized by rewriting the source code into:
77 :
78 : typedef int __attribute__((mode(V8HI))) v8hi;
79 : short a[N]; short b[N]; short c[N]; int i;
80 : v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
81 : v8hi va, vb, vc;
82 :
83 : for (i=0; i<N/8; i++){
84 : vb = pb[i];
85 : vc = pc[i];
86 : va = vb + vc;
87 : pa[i] = va;
88 : }
89 :
90 : The main entry to this pass is vectorize_loops(), in which
91 : the vectorizer applies a set of analyses on a given set of loops,
92 : followed by the actual vectorization transformation for the loops that
93 : had successfully passed the analysis phase.
94 : Throughout this pass we make a distinction between two types of
95 : data: scalars (which are represented by SSA_NAMES), and memory references
96 : ("data-refs"). These two types of data require different handling both
97 : during analysis and transformation. The types of data-refs that the
98 : vectorizer currently supports are ARRAY_REFS which base is an array DECL
99 : (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
100 : accesses are required to have a simple (consecutive) access pattern.
101 :
102 : Analysis phase:
103 : ===============
104 : The driver for the analysis phase is vect_analyze_loop().
105 : It applies a set of analyses, some of which rely on the scalar evolution
106 : analyzer (scev) developed by Sebastian Pop.
107 :
108 : During the analysis phase the vectorizer records some information
109 : per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
110 : loop, as well as general information about the loop as a whole, which is
111 : recorded in a "loop_vec_info" struct attached to each loop.
112 :
113 : Transformation phase:
114 : =====================
115 : The loop transformation phase scans all the stmts in the loop, and
116 : creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
117 : the loop that needs to be vectorized. It inserts the vector code sequence
118 : just before the scalar stmt S, and records a pointer to the vector code
119 : in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
120 : attached to S). This pointer will be used for the vectorization of following
121 : stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
122 : otherwise, we rely on dead code elimination for removing it.
123 :
124 : For example, say stmt S1 was vectorized into stmt VS1:
125 :
126 : VS1: vb = px[i];
127 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
128 : S2: a = b;
129 :
130 : To vectorize stmt S2, the vectorizer first finds the stmt that defines
131 : the operand 'b' (S1), and gets the relevant vector def 'vb' from the
132 : vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
133 : resulting sequence would be:
134 :
135 : VS1: vb = px[i];
136 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
137 : VS2: va = vb;
138 : S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
139 :
140 : Operands that are not SSA_NAMEs, are data-refs that appear in
141 : load/store operations (like 'x[i]' in S1), and are handled differently.
142 :
143 : Target modeling:
144 : =================
145 : Currently the only target specific information that is used is the
146 : size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
147 : Targets that can support different sizes of vectors, for now will need
148 : to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
149 : flexibility will be added in the future.
150 :
151 : Since we only vectorize operations which vector form can be
152 : expressed using existing tree codes, to verify that an operation is
153 : supported, the vectorizer checks the relevant optab at the relevant
154 : machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
155 : the value found is CODE_FOR_nothing, then there's no target support, and
156 : we can't vectorize the stmt.
157 :
158 : For additional information on this project see:
159 : http://gcc.gnu.org/projects/tree-ssa/vectorization.html
160 : */
161 :
162 : static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *,
163 : unsigned *);
164 : static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
165 : gphi **);
166 :
167 :
168 : /* Function vect_is_simple_iv_evolution.
169 :
170 : FORNOW: A simple evolution of an induction variables in the loop is
171 : considered a polynomial evolution. */
172 :
173 : static bool
174 935809 : vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn,
175 : stmt_vec_info stmt_info)
176 : {
177 935809 : tree init_expr;
178 935809 : tree step_expr;
179 935809 : tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
180 935809 : basic_block bb;
181 :
182 : /* When there is no evolution in this loop, the evolution function
183 : is not "simple". */
184 935809 : if (evolution_part == NULL_TREE)
185 : return false;
186 :
187 : /* When the evolution is a polynomial of degree >= 2
188 : the evolution function is not "simple". */
189 823624 : if (tree_is_chrec (evolution_part))
190 : return false;
191 :
192 823624 : step_expr = evolution_part;
193 823624 : init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
194 :
195 823624 : if (dump_enabled_p ())
196 40062 : dump_printf_loc (MSG_NOTE, vect_location, "step: %T, init: %T\n",
197 : step_expr, init_expr);
198 :
199 823624 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
200 823624 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step_expr;
201 :
202 823624 : if (TREE_CODE (step_expr) != INTEGER_CST
203 72409 : && (TREE_CODE (step_expr) != SSA_NAME
204 60488 : || ((bb = gimple_bb (SSA_NAME_DEF_STMT (step_expr)))
205 60225 : && flow_bb_inside_loop_p (get_loop (cfun, loop_nb), bb))
206 7799 : || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
207 133 : && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))
208 133 : || !flag_associative_math)))
209 888331 : && (TREE_CODE (step_expr) != REAL_CST
210 466 : || !flag_associative_math))
211 : {
212 64615 : if (dump_enabled_p ())
213 3130 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
214 : "step unknown.\n");
215 : return false;
216 : }
217 :
218 : return true;
219 : }
220 :
221 : /* Function vect_is_nonlinear_iv_evolution
222 :
223 : Only support nonlinear induction for integer type
224 : 1. neg
225 : 2. mul by constant
226 : 3. lshift/rshift by constant.
227 :
228 : For neg induction, return a fake step as integer -1. */
229 : static bool
230 174231 : vect_is_nonlinear_iv_evolution (class loop* loop, stmt_vec_info stmt_info,
231 : gphi* loop_phi_node)
232 : {
233 174231 : tree init_expr, ev_expr, result, op1, op2;
234 174231 : gimple* def;
235 :
236 174231 : if (gimple_phi_num_args (loop_phi_node) != 2)
237 : return false;
238 :
239 174231 : init_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_preheader_edge (loop));
240 174231 : ev_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_latch_edge (loop));
241 :
242 : /* Support nonlinear induction only for integer type. */
243 174231 : if (!INTEGRAL_TYPE_P (TREE_TYPE (init_expr)))
244 : return false;
245 :
246 109735 : result = PHI_RESULT (loop_phi_node);
247 :
248 109735 : if (TREE_CODE (ev_expr) != SSA_NAME
249 107399 : || ((def = SSA_NAME_DEF_STMT (ev_expr)), false)
250 109735 : || !is_gimple_assign (def))
251 : return false;
252 :
253 98856 : enum tree_code t_code = gimple_assign_rhs_code (def);
254 98856 : tree step;
255 98856 : switch (t_code)
256 : {
257 3540 : case NEGATE_EXPR:
258 3540 : if (gimple_assign_rhs1 (def) != result)
259 : return false;
260 3540 : step = build_int_cst (TREE_TYPE (init_expr), -1);
261 3540 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_neg;
262 3540 : break;
263 :
264 11827 : case RSHIFT_EXPR:
265 11827 : case LSHIFT_EXPR:
266 11827 : case MULT_EXPR:
267 11827 : op1 = gimple_assign_rhs1 (def);
268 11827 : op2 = gimple_assign_rhs2 (def);
269 11827 : if (TREE_CODE (op2) != INTEGER_CST
270 7933 : || op1 != result)
271 : return false;
272 7538 : step = op2;
273 7538 : if (t_code == LSHIFT_EXPR)
274 472 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shl;
275 7066 : else if (t_code == RSHIFT_EXPR)
276 6084 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shr;
277 : /* NEGATE_EXPR and MULT_EXPR are both vect_step_op_mul. */
278 : else
279 982 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_mul;
280 : break;
281 :
282 : default:
283 : return false;
284 : }
285 :
286 11078 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
287 11078 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step;
288 :
289 11078 : return true;
290 : }
291 :
292 : /* Returns true if Phi is a first-order recurrence. A first-order
293 : recurrence is a non-reduction recurrence relation in which the value of
294 : the recurrence in the current loop iteration equals a value defined in
295 : the previous iteration. */
296 :
297 : static bool
298 64312 : vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo, class loop *loop,
299 : gphi *phi)
300 : {
301 : /* A nested cycle isn't vectorizable as first order recurrence. */
302 64312 : if (LOOP_VINFO_LOOP (loop_vinfo) != loop)
303 : return false;
304 :
305 : /* Ensure the loop latch definition is from within the loop. */
306 64170 : edge latch = loop_latch_edge (loop);
307 64170 : tree ldef = PHI_ARG_DEF_FROM_EDGE (phi, latch);
308 64170 : if (TREE_CODE (ldef) != SSA_NAME
309 61491 : || SSA_NAME_IS_DEFAULT_DEF (ldef)
310 61425 : || is_a <gphi *> (SSA_NAME_DEF_STMT (ldef))
311 120884 : || !flow_bb_inside_loop_p (loop, gimple_bb (SSA_NAME_DEF_STMT (ldef))))
312 : return false;
313 :
314 56053 : tree def = gimple_phi_result (phi);
315 :
316 : /* Ensure every use_stmt of the phi node is dominated by the latch
317 : definition. */
318 56053 : imm_use_iterator imm_iter;
319 56053 : use_operand_p use_p;
320 71756 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, def)
321 71231 : if (!is_gimple_debug (USE_STMT (use_p))
322 135735 : && (SSA_NAME_DEF_STMT (ldef) == USE_STMT (use_p)
323 43597 : || !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (ldef),
324 : USE_STMT (use_p))))
325 55528 : return false;
326 :
327 : /* First-order recurrence autovectorization needs shuffle vector. */
328 525 : tree scalar_type = TREE_TYPE (def);
329 525 : tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
330 525 : if (!vectype)
331 6 : return false;
332 :
333 : return true;
334 : }
335 :
336 : /* Function vect_analyze_scalar_cycles_1.
337 :
338 : Examine the cross iteration def-use cycles of scalar variables
339 : in LOOP. LOOP_VINFO represents the loop that is now being
340 : considered for vectorization (can be LOOP, or an outer-loop
341 : enclosing LOOP). SLP indicates there will be some subsequent
342 : slp analyses or not. */
343 :
344 : static void
345 457909 : vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
346 : {
347 457909 : basic_block bb = loop->header;
348 457909 : auto_vec<stmt_vec_info, 64> worklist;
349 457909 : gphi_iterator gsi;
350 :
351 457909 : DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
352 :
353 : /* First - identify all inductions. Reduction detection assumes that all the
354 : inductions have been identified, therefore, this order must not be
355 : changed. */
356 1639409 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
357 : {
358 1181500 : gphi *phi = gsi.phi ();
359 1181500 : tree access_fn = NULL;
360 1181500 : tree def = PHI_RESULT (phi);
361 1181500 : stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi);
362 :
363 : /* Skip virtual phi's. The data dependences that are associated with
364 : virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
365 2363000 : if (virtual_operand_p (def))
366 411419 : continue;
367 :
368 : /* Skip already analyzed inner loop PHIs of double reductions. */
369 936814 : if (VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
370 1005 : continue;
371 :
372 935809 : if (dump_enabled_p ())
373 42185 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
374 : (gimple *) phi);
375 :
376 935809 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
377 :
378 : /* Analyze the evolution function. */
379 935809 : access_fn = analyze_scalar_evolution (loop, def);
380 935809 : if (dump_enabled_p ())
381 42185 : dump_printf_loc (MSG_NOTE, vect_location,
382 : "Access function of PHI: %T\n", access_fn);
383 935809 : if (access_fn)
384 935809 : STRIP_NOPS (access_fn);
385 :
386 1101537 : if ((!access_fn
387 935809 : || !vect_is_simple_iv_evolution (loop->num, access_fn, stmt_vinfo)
388 759009 : || (LOOP_VINFO_LOOP (loop_vinfo) != loop
389 11436 : && (TREE_CODE (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo))
390 : != INTEGER_CST)))
391 : /* Only handle nonlinear iv for same loop. */
392 1112615 : && (LOOP_VINFO_LOOP (loop_vinfo) != loop
393 174231 : || !vect_is_nonlinear_iv_evolution (loop, stmt_vinfo, phi)))
394 : {
395 165728 : worklist.safe_push (stmt_vinfo);
396 165728 : continue;
397 : }
398 :
399 770081 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo)
400 : != NULL_TREE);
401 770081 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo) != NULL_TREE);
402 :
403 770081 : if (dump_enabled_p ())
404 37041 : dump_printf_loc (MSG_NOTE, vect_location, "Detected induction.\n");
405 770081 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
406 :
407 : /* Mark if we have a non-linear IV. */
408 770081 : LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
409 770081 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add;
410 : }
411 :
412 :
413 : /* Second - identify all reductions and nested cycles. */
414 623637 : while (worklist.length () > 0)
415 : {
416 165728 : stmt_vec_info stmt_vinfo = worklist.pop ();
417 165728 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
418 165728 : tree def = PHI_RESULT (phi);
419 :
420 165728 : if (dump_enabled_p ())
421 5144 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
422 : (gimple *) phi);
423 :
424 331456 : gcc_assert (!virtual_operand_p (def)
425 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
426 :
427 165728 : gphi *double_reduc;
428 165728 : stmt_vec_info reduc_stmt_info
429 165728 : = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc);
430 165728 : if (reduc_stmt_info && double_reduc)
431 : {
432 1107 : stmt_vec_info inner_phi_info
433 1107 : = loop_vinfo->lookup_stmt (double_reduc);
434 : /* ??? Pass down flag we're the inner loop of a double reduc. */
435 1107 : stmt_vec_info inner_reduc_info
436 1107 : = vect_is_simple_reduction (loop_vinfo, inner_phi_info, NULL);
437 1107 : if (inner_reduc_info)
438 : {
439 1005 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
440 1005 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
441 1005 : STMT_VINFO_REDUC_DEF (inner_phi_info) = inner_reduc_info;
442 1005 : STMT_VINFO_REDUC_DEF (inner_reduc_info) = inner_phi_info;
443 1005 : if (dump_enabled_p ())
444 130 : dump_printf_loc (MSG_NOTE, vect_location,
445 : "Detected double reduction.\n");
446 :
447 1005 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
448 1005 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_double_reduction_def;
449 1005 : STMT_VINFO_DEF_TYPE (inner_phi_info) = vect_nested_cycle;
450 : /* Make it accessible for SLP vectorization. */
451 1005 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
452 : }
453 102 : else if (dump_enabled_p ())
454 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
455 : "Unknown def-use cycle pattern.\n");
456 : }
457 164621 : else if (reduc_stmt_info)
458 : {
459 100309 : if (loop != LOOP_VINFO_LOOP (loop_vinfo))
460 : {
461 2433 : if (dump_enabled_p ())
462 434 : dump_printf_loc (MSG_NOTE, vect_location,
463 : "Detected vectorizable nested cycle.\n");
464 :
465 2433 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
466 : }
467 : else
468 : {
469 97876 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
470 97876 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
471 97876 : if (dump_enabled_p ())
472 4004 : dump_printf_loc (MSG_NOTE, vect_location,
473 : "Detected reduction.\n");
474 :
475 97876 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
476 97876 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def;
477 97876 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
478 : }
479 : }
480 64312 : else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
481 519 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
482 : else
483 63793 : if (dump_enabled_p ())
484 483 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
485 : "Unknown def-use cycle pattern.\n");
486 : }
487 457909 : }
488 :
489 :
490 : /* Function vect_analyze_scalar_cycles.
491 :
492 : Examine the cross iteration def-use cycles of scalar variables, by
493 : analyzing the loop-header PHIs of scalar variables. Classify each
494 : cycle as one of the following: invariant, induction, reduction, unknown.
495 : We do that for the loop represented by LOOP_VINFO, and also to its
496 : inner-loop, if exists.
497 : Examples for scalar cycles:
498 :
499 : Example1: reduction:
500 :
501 : loop1:
502 : for (i=0; i<N; i++)
503 : sum += a[i];
504 :
505 : Example2: induction:
506 :
507 : loop2:
508 : for (i=0; i<N; i++)
509 : a[i] = i; */
510 :
511 : static void
512 452094 : vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
513 : {
514 452094 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
515 :
516 452094 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
517 :
518 : /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
519 : Reductions in such inner-loop therefore have different properties than
520 : the reductions in the nest that gets vectorized:
521 : 1. When vectorized, they are executed in the same order as in the original
522 : scalar loop, so we can't change the order of computation when
523 : vectorizing them.
524 : 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
525 : current checks are too strict. */
526 :
527 452094 : if (loop->inner)
528 5815 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
529 452094 : }
530 :
531 : /* Function vect_get_loop_niters.
532 :
533 : Determine how many iterations the loop is executed and place it
534 : in NUMBER_OF_ITERATIONS. Place the number of latch iterations
535 : in NUMBER_OF_ITERATIONSM1. Place the condition under which the
536 : niter information holds in ASSUMPTIONS.
537 :
538 : Return the loop exit conditions. */
539 :
540 :
541 : static vec<gcond *>
542 284651 : vect_get_loop_niters (class loop *loop, const_edge main_exit, tree *assumptions,
543 : tree *number_of_iterations, tree *number_of_iterationsm1)
544 : {
545 284651 : auto_vec<edge> exits = get_loop_exit_edges (loop);
546 284651 : vec<gcond *> conds;
547 569302 : conds.create (exits.length ());
548 284651 : class tree_niter_desc niter_desc;
549 284651 : tree niter_assumptions, niter, may_be_zero;
550 :
551 284651 : *assumptions = boolean_true_node;
552 284651 : *number_of_iterationsm1 = chrec_dont_know;
553 284651 : *number_of_iterations = chrec_dont_know;
554 :
555 284651 : DUMP_VECT_SCOPE ("get_loop_niters");
556 :
557 284651 : if (exits.is_empty ())
558 0 : return conds;
559 :
560 284651 : if (dump_enabled_p ())
561 14699 : dump_printf_loc (MSG_NOTE, vect_location, "Loop has %d exits.\n",
562 : exits.length ());
563 :
564 284651 : edge exit;
565 284651 : unsigned int i;
566 694634 : FOR_EACH_VEC_ELT (exits, i, exit)
567 : {
568 409983 : gcond *cond = get_loop_exit_condition (exit);
569 409983 : if (cond)
570 409949 : conds.safe_push (cond);
571 :
572 409983 : if (dump_enabled_p ())
573 15861 : dump_printf_loc (MSG_NOTE, vect_location, "Analyzing exit %d...\n", i);
574 :
575 409983 : if (exit != main_exit)
576 185275 : continue;
577 :
578 284651 : may_be_zero = NULL_TREE;
579 284651 : if (!number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
580 284651 : || chrec_contains_undetermined (niter_desc.niter))
581 59943 : continue;
582 :
583 224708 : niter_assumptions = niter_desc.assumptions;
584 224708 : may_be_zero = niter_desc.may_be_zero;
585 224708 : niter = niter_desc.niter;
586 :
587 224708 : if (may_be_zero && integer_zerop (may_be_zero))
588 : may_be_zero = NULL_TREE;
589 :
590 9466 : if (may_be_zero)
591 : {
592 9466 : if (COMPARISON_CLASS_P (may_be_zero))
593 : {
594 : /* Try to combine may_be_zero with assumptions, this can simplify
595 : computation of niter expression. */
596 9466 : if (niter_assumptions && !integer_nonzerop (niter_assumptions))
597 968 : niter_assumptions = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
598 : niter_assumptions,
599 : fold_build1 (TRUTH_NOT_EXPR,
600 : boolean_type_node,
601 : may_be_zero));
602 : else
603 8498 : niter = fold_build3 (COND_EXPR, TREE_TYPE (niter), may_be_zero,
604 : build_int_cst (TREE_TYPE (niter), 0),
605 : rewrite_to_non_trapping_overflow (niter));
606 :
607 224708 : may_be_zero = NULL_TREE;
608 : }
609 0 : else if (integer_nonzerop (may_be_zero))
610 : {
611 0 : *number_of_iterationsm1 = build_int_cst (TREE_TYPE (niter), 0);
612 0 : *number_of_iterations = build_int_cst (TREE_TYPE (niter), 1);
613 0 : continue;
614 : }
615 : else
616 0 : continue;
617 : }
618 :
619 : /* Loop assumptions are based off the normal exit. */
620 224708 : *assumptions = niter_assumptions;
621 224708 : *number_of_iterationsm1 = niter;
622 :
623 : /* We want the number of loop header executions which is the number
624 : of latch executions plus one.
625 : ??? For UINT_MAX latch executions this number overflows to zero
626 : for loops like do { n++; } while (n != 0); */
627 224708 : if (niter && !chrec_contains_undetermined (niter))
628 : {
629 224708 : niter = fold_build2 (PLUS_EXPR, TREE_TYPE (niter),
630 : unshare_expr (niter),
631 : build_int_cst (TREE_TYPE (niter), 1));
632 224708 : if (TREE_CODE (niter) == INTEGER_CST
633 124221 : && TREE_CODE (*number_of_iterationsm1) != INTEGER_CST)
634 : {
635 : /* If we manage to fold niter + 1 into INTEGER_CST even when
636 : niter is some complex expression, ensure back
637 : *number_of_iterationsm1 is an INTEGER_CST as well. See
638 : PR113210. */
639 0 : *number_of_iterationsm1
640 0 : = fold_build2 (PLUS_EXPR, TREE_TYPE (niter), niter,
641 : build_minus_one_cst (TREE_TYPE (niter)));
642 : }
643 : }
644 224708 : *number_of_iterations = niter;
645 : }
646 :
647 284651 : if (dump_enabled_p ())
648 14699 : dump_printf_loc (MSG_NOTE, vect_location, "All loop exits successfully analyzed.\n");
649 :
650 284651 : return conds;
651 284651 : }
652 :
653 : /* Determine the main loop exit for the vectorizer. */
654 :
655 : edge
656 501543 : vec_init_loop_exit_info (class loop *loop)
657 : {
658 : /* Before we begin we must first determine which exit is the main one and
659 : which are auxiliary exits. */
660 501543 : auto_vec<edge> exits = get_loop_exit_edges (loop);
661 998004 : if (exits.length () == 0)
662 : return NULL;
663 496461 : if (exits.length () == 1)
664 324996 : return exits[0];
665 :
666 : /* If we have multiple exits, look for counting IV exit.
667 : Analyze all exits and return the last one we can analyze. */
668 171465 : class tree_niter_desc niter_desc;
669 171465 : edge candidate = NULL;
670 636558 : for (edge exit : exits)
671 : {
672 486405 : if (!get_loop_exit_condition (exit))
673 : {
674 21312 : if (dump_enabled_p ())
675 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
676 : "Unhandled loop exit detected.\n");
677 : return NULL;
678 : }
679 :
680 465093 : if (number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
681 465093 : && !chrec_contains_undetermined (niter_desc.niter))
682 : {
683 137283 : tree may_be_zero = niter_desc.may_be_zero;
684 137283 : if ((integer_zerop (may_be_zero)
685 : /* As we are handling may_be_zero that's not false by
686 : rewriting niter to may_be_zero ? 0 : niter we require
687 : an empty latch. */
688 475879 : || (single_pred_p (loop->latch)
689 10464 : && exit->src == single_pred (loop->latch)
690 2773 : && (integer_nonzerop (may_be_zero)
691 2773 : || COMPARISON_CLASS_P (may_be_zero))))
692 140056 : && (!candidate
693 6008 : || dominated_by_p (CDI_DOMINATORS, exit->src,
694 6008 : candidate->src)))
695 : candidate = exit;
696 : }
697 : }
698 :
699 : /* If no exit is analyzable by scalar evolution, we return the last exit
700 : under the assummption we are dealing with an uncounted loop. */
701 207475 : if (!candidate && single_pred_p (loop->latch))
702 36010 : candidate = loop_exits_from_bb_p (loop, single_pred (loop->latch));
703 :
704 : return candidate;
705 171465 : }
706 :
707 : /* Function bb_in_loop_p
708 :
709 : Used as predicate for dfs order traversal of the loop bbs. */
710 :
711 : static bool
712 1727218 : bb_in_loop_p (const_basic_block bb, const void *data)
713 : {
714 1727218 : const class loop *const loop = (const class loop *)data;
715 1727218 : if (flow_bb_inside_loop_p (loop, bb))
716 : return true;
717 : return false;
718 : }
719 :
720 :
721 : /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
722 : stmt_vec_info structs for all the stmts in LOOP_IN. */
723 :
724 589396 : _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
725 : : vec_info (vec_info::loop, shared),
726 589396 : loop (loop_in),
727 589396 : num_itersm1 (NULL_TREE),
728 589396 : num_iters (NULL_TREE),
729 589396 : num_iters_unchanged (NULL_TREE),
730 589396 : num_iters_assumptions (NULL_TREE),
731 589396 : vector_costs (nullptr),
732 589396 : scalar_costs (nullptr),
733 589396 : th (0),
734 589396 : versioning_threshold (0),
735 589396 : vectorization_factor (0),
736 589396 : main_loop_edge (nullptr),
737 589396 : skip_main_loop_edge (nullptr),
738 589396 : skip_this_loop_edge (nullptr),
739 589396 : reusable_accumulators (),
740 589396 : suggested_unroll_factor (1),
741 589396 : max_vectorization_factor (0),
742 589396 : mask_skip_niters (NULL_TREE),
743 589396 : mask_skip_niters_pfa_offset (NULL_TREE),
744 589396 : rgroup_compare_type (NULL_TREE),
745 589396 : simd_if_cond (NULL_TREE),
746 589396 : partial_vector_style (vect_partial_vectors_none),
747 589396 : unaligned_dr (NULL),
748 589396 : peeling_for_alignment (0),
749 589396 : ptr_mask (0),
750 589396 : max_spec_read_amount (0),
751 589396 : nonlinear_iv (false),
752 589396 : ivexpr_map (NULL),
753 589396 : scan_map (NULL),
754 589396 : inner_loop_cost_factor (param_vect_inner_loop_cost_factor),
755 589396 : vectorizable (false),
756 589396 : can_use_partial_vectors_p (true),
757 589396 : must_use_partial_vectors_p (false),
758 589396 : using_partial_vectors_p (false),
759 589396 : using_decrementing_iv_p (false),
760 589396 : using_select_vl_p (false),
761 589396 : allow_mutual_alignment (false),
762 589396 : partial_load_store_bias (0),
763 589396 : peeling_for_gaps (false),
764 589396 : peeling_for_niter (false),
765 589396 : early_breaks (false),
766 589396 : loop_iv_cond (NULL),
767 589396 : user_unroll (false),
768 589396 : no_data_dependencies (false),
769 589396 : has_mask_store (false),
770 589396 : scalar_loop_scaling (profile_probability::uninitialized ()),
771 589396 : scalar_loop (NULL),
772 589396 : main_loop_info (NULL),
773 589396 : orig_loop_info (NULL),
774 589396 : epilogue_vinfo (NULL),
775 589396 : drs_advanced_by (NULL_TREE),
776 589396 : vec_loop_main_exit (NULL),
777 589396 : vec_epilogue_loop_main_exit (NULL),
778 589396 : scalar_loop_main_exit (NULL),
779 589396 : early_break_needs_epilogue (false),
780 589396 : early_break_niters_var (NULL)
781 : {
782 : /* CHECKME: We want to visit all BBs before their successors (except for
783 : latch blocks, for which this assertion wouldn't hold). In the simple
784 : case of the loop forms we allow, a dfs order of the BBs would the same
785 : as reversed postorder traversal, so we are safe. */
786 :
787 589396 : bbs = XCNEWVEC (basic_block, loop->num_nodes);
788 1178792 : nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p, bbs,
789 589396 : loop->num_nodes, loop);
790 589396 : gcc_assert (nbbs == loop->num_nodes);
791 :
792 2056066 : for (unsigned int i = 0; i < nbbs; i++)
793 : {
794 1466670 : basic_block bb = bbs[i];
795 1466670 : gimple_stmt_iterator si;
796 :
797 3027345 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
798 : {
799 1560675 : gimple *phi = gsi_stmt (si);
800 1560675 : gimple_set_uid (phi, 0);
801 1560675 : add_stmt (phi);
802 : }
803 :
804 13681101 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
805 : {
806 10747761 : gimple *stmt = gsi_stmt (si);
807 10747761 : gimple_set_uid (stmt, 0);
808 10747761 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
809 4662005 : continue;
810 6085756 : add_stmt (stmt);
811 : /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
812 : third argument is the #pragma omp simd if (x) condition, when 0,
813 : loop shouldn't be vectorized, when non-zero constant, it should
814 : be vectorized normally, otherwise versioned with vectorized loop
815 : done if the condition is non-zero at runtime. */
816 6085756 : if (loop_in->simduid
817 43427 : && is_gimple_call (stmt)
818 4265 : && gimple_call_internal_p (stmt)
819 4138 : && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
820 4134 : && gimple_call_num_args (stmt) >= 3
821 103 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
822 6085859 : && (loop_in->simduid
823 103 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0))))
824 : {
825 103 : tree arg = gimple_call_arg (stmt, 2);
826 103 : if (integer_zerop (arg) || TREE_CODE (arg) == SSA_NAME)
827 103 : simd_if_cond = arg;
828 : else
829 0 : gcc_assert (integer_nonzerop (arg));
830 : }
831 : }
832 : }
833 589396 : }
834 :
835 : /* Free all levels of rgroup CONTROLS. */
836 :
837 : void
838 1474106 : release_vec_loop_controls (vec<rgroup_controls> *controls)
839 : {
840 1474106 : rgroup_controls *rgc;
841 1474106 : unsigned int i;
842 1498705 : FOR_EACH_VEC_ELT (*controls, i, rgc)
843 24599 : rgc->controls.release ();
844 1474106 : controls->release ();
845 1474106 : }
846 :
847 : /* Free all memory used by the _loop_vec_info, as well as all the
848 : stmt_vec_info structs of all the stmts in the loop. */
849 :
850 589396 : _loop_vec_info::~_loop_vec_info ()
851 : {
852 589396 : free (bbs);
853 :
854 589396 : release_vec_loop_controls (&masks.rgc_vec);
855 589396 : release_vec_loop_controls (&lens);
856 593302 : delete ivexpr_map;
857 589718 : delete scan_map;
858 589396 : delete scalar_costs;
859 589396 : delete vector_costs;
860 806040 : for (auto reduc_info : reduc_infos)
861 207808 : delete reduc_info;
862 :
863 : /* When we release an epiloge vinfo that we do not intend to use
864 : avoid clearing AUX of the main loop which should continue to
865 : point to the main loop vinfo since otherwise we'll leak that. */
866 589396 : if (loop->aux == this)
867 62148 : loop->aux = NULL;
868 1178792 : }
869 :
870 : /* Return an invariant or register for EXPR and emit necessary
871 : computations in the LOOP_VINFO loop preheader. */
872 :
873 : tree
874 20430 : cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo, tree expr)
875 : {
876 20430 : if (is_gimple_reg (expr)
877 20430 : || is_gimple_min_invariant (expr))
878 : return expr;
879 :
880 13474 : if (! loop_vinfo->ivexpr_map)
881 3906 : loop_vinfo->ivexpr_map = new hash_map<tree_operand_hash, tree>;
882 13474 : tree &cached = loop_vinfo->ivexpr_map->get_or_insert (expr);
883 13474 : if (! cached)
884 : {
885 8648 : gimple_seq stmts = NULL;
886 8648 : cached = force_gimple_operand (unshare_expr (expr),
887 : &stmts, true, NULL_TREE);
888 8648 : if (stmts)
889 : {
890 8500 : edge e = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
891 8500 : gsi_insert_seq_on_edge_immediate (e, stmts);
892 : }
893 : }
894 13474 : return cached;
895 : }
896 :
897 : /* Return true if we can use CMP_TYPE as the comparison type to produce
898 : all masks required to mask LOOP_VINFO. */
899 :
900 : static bool
901 110198 : can_produce_all_loop_masks_p (loop_vec_info loop_vinfo, tree cmp_type)
902 : {
903 110198 : rgroup_controls *rgm;
904 110198 : unsigned int i;
905 125912 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
906 125912 : if (rgm->type != NULL_TREE
907 125912 : && !direct_internal_fn_supported_p (IFN_WHILE_ULT,
908 : cmp_type, rgm->type,
909 : OPTIMIZE_FOR_SPEED))
910 : return false;
911 : return true;
912 : }
913 :
914 : /* Calculate the maximum number of scalars per iteration for every
915 : rgroup in LOOP_VINFO. */
916 :
917 : static unsigned int
918 23482 : vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo)
919 : {
920 23482 : unsigned int res = 1;
921 23482 : unsigned int i;
922 23482 : rgroup_controls *rgm;
923 56274 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
924 32792 : res = MAX (res, rgm->max_nscalars_per_iter);
925 23482 : return res;
926 : }
927 :
928 : /* Calculate the minimum precision necessary to represent:
929 :
930 : MAX_NITERS * FACTOR
931 :
932 : as an unsigned integer, where MAX_NITERS is the maximum number of
933 : loop header iterations for the original scalar form of LOOP_VINFO. */
934 :
935 : unsigned
936 25920 : vect_min_prec_for_max_niters (loop_vec_info loop_vinfo, unsigned int factor)
937 : {
938 25920 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
939 :
940 : /* Get the maximum number of iterations that is representable
941 : in the counter type. */
942 25920 : tree ni_type;
943 25920 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
944 25920 : ni_type = TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo));
945 : else
946 0 : ni_type = sizetype;
947 25920 : widest_int max_ni = wi::to_widest (TYPE_MAX_VALUE (ni_type)) + 1;
948 :
949 : /* Get a more refined estimate for the number of iterations. */
950 25920 : widest_int max_back_edges;
951 25920 : if (max_loop_iterations (loop, &max_back_edges))
952 25920 : max_ni = wi::smin (max_ni, max_back_edges + 1);
953 :
954 : /* Work out how many bits we need to represent the limit. */
955 25920 : return wi::min_precision (max_ni * factor, UNSIGNED);
956 25920 : }
957 :
958 : /* True if the loop needs peeling or partial vectors when vectorized. */
959 :
960 : static bool
961 156564 : vect_need_peeling_or_partial_vectors_p (loop_vec_info loop_vinfo)
962 : {
963 156564 : unsigned HOST_WIDE_INT const_vf;
964 :
965 156564 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
966 : return true;
967 :
968 13311 : loop_vec_info main_loop_vinfo
969 155259 : = (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
970 155259 : ? LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) : loop_vinfo);
971 155259 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
972 79787 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo) >= 0)
973 : {
974 : /* Work out the (constant) number of iterations that need to be
975 : peeled for reasons other than niters. */
976 79737 : unsigned int peel_niter
977 : = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
978 79737 : return !multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter,
979 79737 : LOOP_VINFO_VECT_FACTOR (loop_vinfo));
980 : }
981 :
982 75522 : if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo)
983 75522 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&const_vf))
984 : {
985 : /* When the number of iterations is a multiple of the vectorization
986 : factor and we are not doing prologue or forced epilogue peeling
987 : the epilogue isn't necessary. */
988 75105 : if (tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
989 150210 : >= (unsigned) exact_log2 (const_vf))
990 1753 : return false;
991 : }
992 :
993 : return true;
994 : }
995 :
996 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
997 : whether we can actually generate the masks required. Return true if so,
998 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_COMPARE_TYPE. */
999 :
1000 : static bool
1001 23482 : vect_verify_full_masking (loop_vec_info loop_vinfo)
1002 : {
1003 23482 : unsigned int min_ni_width;
1004 :
1005 : /* Use a normal loop if there are no statements that need masking.
1006 : This only happens in rare degenerate cases: it means that the loop
1007 : has no loads, no stores, and no live-out values. */
1008 23482 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1009 : return false;
1010 :
1011 : /* Produce the rgroup controls. */
1012 58021 : for (auto mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1013 : {
1014 34539 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1015 34539 : tree vectype = mask.first;
1016 34539 : unsigned nvectors = mask.second;
1017 :
1018 45596 : if (masks->rgc_vec.length () < nvectors)
1019 25688 : masks->rgc_vec.safe_grow_cleared (nvectors, true);
1020 34539 : rgroup_controls *rgm = &(*masks).rgc_vec[nvectors - 1];
1021 : /* The number of scalars per iteration and the number of vectors are
1022 : both compile-time constants. */
1023 34539 : unsigned int nscalars_per_iter
1024 34539 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1025 34539 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1026 :
1027 34539 : if (rgm->max_nscalars_per_iter < nscalars_per_iter)
1028 : {
1029 27521 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1030 27521 : rgm->type = truth_type_for (vectype);
1031 27521 : rgm->factor = 1;
1032 : }
1033 : }
1034 :
1035 23482 : unsigned int max_nscalars_per_iter
1036 23482 : = vect_get_max_nscalars_per_iter (loop_vinfo);
1037 :
1038 : /* Work out how many bits we need to represent the limit. */
1039 23482 : min_ni_width
1040 23482 : = vect_min_prec_for_max_niters (loop_vinfo, max_nscalars_per_iter);
1041 :
1042 : /* Find a scalar mode for which WHILE_ULT is supported. */
1043 23482 : opt_scalar_int_mode cmp_mode_iter;
1044 23482 : tree cmp_type = NULL_TREE;
1045 23482 : tree iv_type = NULL_TREE;
1046 23482 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1047 23482 : unsigned int iv_precision = UINT_MAX;
1048 :
1049 23482 : if (iv_limit != -1)
1050 23482 : iv_precision = wi::min_precision (iv_limit * max_nscalars_per_iter,
1051 : UNSIGNED);
1052 :
1053 187856 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1054 : {
1055 164374 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1056 164374 : if (cmp_bits >= min_ni_width
1057 164374 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1058 : {
1059 110198 : tree this_type = build_nonstandard_integer_type (cmp_bits, true);
1060 110198 : if (this_type
1061 110198 : && can_produce_all_loop_masks_p (loop_vinfo, this_type))
1062 : {
1063 : /* Although we could stop as soon as we find a valid mode,
1064 : there are at least two reasons why that's not always the
1065 : best choice:
1066 :
1067 : - An IV that's Pmode or wider is more likely to be reusable
1068 : in address calculations than an IV that's narrower than
1069 : Pmode.
1070 :
1071 : - Doing the comparison in IV_PRECISION or wider allows
1072 : a natural 0-based IV, whereas using a narrower comparison
1073 : type requires mitigations against wrap-around.
1074 :
1075 : Conversely, if the IV limit is variable, doing the comparison
1076 : in a wider type than the original type can introduce
1077 : unnecessary extensions, so picking the widest valid mode
1078 : is not always a good choice either.
1079 :
1080 : Here we prefer the first IV type that's Pmode or wider,
1081 : and the first comparison type that's IV_PRECISION or wider.
1082 : (The comparison type must be no wider than the IV type,
1083 : to avoid extensions in the vector loop.)
1084 :
1085 : ??? We might want to try continuing beyond Pmode for ILP32
1086 : targets if CMP_BITS < IV_PRECISION. */
1087 0 : iv_type = this_type;
1088 0 : if (!cmp_type || iv_precision > TYPE_PRECISION (cmp_type))
1089 : cmp_type = this_type;
1090 0 : if (cmp_bits >= GET_MODE_BITSIZE (Pmode))
1091 : break;
1092 : }
1093 : }
1094 : }
1095 :
1096 23482 : if (!cmp_type)
1097 : {
1098 23482 : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.release ();
1099 23482 : return false;
1100 : }
1101 :
1102 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
1103 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1104 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_while_ult;
1105 0 : return true;
1106 23482 : }
1107 :
1108 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
1109 : whether we can actually generate AVX512 style masks. Return true if so,
1110 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_IV_TYPE. */
1111 :
1112 : static bool
1113 23482 : vect_verify_full_masking_avx512 (loop_vec_info loop_vinfo)
1114 : {
1115 : /* Produce differently organized rgc_vec and differently check
1116 : we can produce masks. */
1117 :
1118 : /* Use a normal loop if there are no statements that need masking.
1119 : This only happens in rare degenerate cases: it means that the loop
1120 : has no loads, no stores, and no live-out values. */
1121 23482 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1122 : return false;
1123 :
1124 : /* For the decrementing IV we need to represent all values in
1125 : [0, niter + niter_skip] where niter_skip is the elements we
1126 : skip in the first iteration for prologue peeling. */
1127 23482 : tree iv_type = NULL_TREE;
1128 23482 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1129 23482 : unsigned int iv_precision = UINT_MAX;
1130 23482 : if (iv_limit != -1)
1131 23482 : iv_precision = wi::min_precision (iv_limit, UNSIGNED);
1132 :
1133 : /* First compute the type for the IV we use to track the remaining
1134 : scalar iterations. */
1135 23482 : opt_scalar_int_mode cmp_mode_iter;
1136 30669 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1137 : {
1138 30669 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1139 30669 : if (cmp_bits >= iv_precision
1140 30669 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1141 : {
1142 23482 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
1143 23482 : if (iv_type)
1144 : break;
1145 : }
1146 : }
1147 23482 : if (!iv_type)
1148 : return false;
1149 :
1150 : /* Produce the rgroup controls. */
1151 58021 : for (auto const &mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1152 : {
1153 34539 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1154 34539 : tree vectype = mask.first;
1155 34539 : unsigned nvectors = mask.second;
1156 :
1157 : /* The number of scalars per iteration and the number of vectors are
1158 : both compile-time constants. */
1159 34539 : unsigned int nscalars_per_iter
1160 34539 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1161 34539 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1162 :
1163 : /* We index the rgroup_controls vector with nscalars_per_iter
1164 : which we keep constant and instead have a varying nvectors,
1165 : remembering the vector mask with the fewest nV. */
1166 45596 : if (masks->rgc_vec.length () < nscalars_per_iter)
1167 23555 : masks->rgc_vec.safe_grow_cleared (nscalars_per_iter, true);
1168 34539 : rgroup_controls *rgm = &(*masks).rgc_vec[nscalars_per_iter - 1];
1169 :
1170 34539 : if (!rgm->type || rgm->factor > nvectors)
1171 : {
1172 25382 : rgm->type = truth_type_for (vectype);
1173 25382 : rgm->compare_type = NULL_TREE;
1174 25382 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1175 25382 : rgm->factor = nvectors;
1176 25382 : rgm->bias_adjusted_ctrl = NULL_TREE;
1177 : }
1178 : }
1179 :
1180 : /* There is no fixed compare type we are going to use but we have to
1181 : be able to get at one for each mask group. */
1182 23482 : unsigned int min_ni_width
1183 23482 : = wi::min_precision (vect_max_vf (loop_vinfo), UNSIGNED);
1184 :
1185 23482 : bool ok = true;
1186 88905 : for (auto &rgc : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
1187 : {
1188 24551 : tree mask_type = rgc.type;
1189 24551 : if (!mask_type)
1190 990 : continue;
1191 :
1192 : /* For now vect_get_loop_mask only supports integer mode masks
1193 : when we need to split it. */
1194 23561 : if (GET_MODE_CLASS (TYPE_MODE (mask_type)) != MODE_INT
1195 23561 : || TYPE_PRECISION (TREE_TYPE (mask_type)) != 1)
1196 : {
1197 : ok = false;
1198 : break;
1199 : }
1200 :
1201 : /* If iv_type is usable as compare type use that - we can elide the
1202 : saturation in that case. */
1203 17473 : if (TYPE_PRECISION (iv_type) >= min_ni_width)
1204 : {
1205 17473 : tree cmp_vectype
1206 17473 : = build_vector_type (iv_type, TYPE_VECTOR_SUBPARTS (mask_type));
1207 17473 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1208 5943 : rgc.compare_type = cmp_vectype;
1209 : }
1210 17473 : if (!rgc.compare_type)
1211 33173 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1212 : {
1213 33169 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1214 33169 : if (cmp_bits >= min_ni_width
1215 33169 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1216 : {
1217 33157 : tree cmp_type = build_nonstandard_integer_type (cmp_bits, true);
1218 33157 : if (!cmp_type)
1219 0 : continue;
1220 :
1221 : /* Check whether we can produce the mask with cmp_type. */
1222 33157 : tree cmp_vectype
1223 33157 : = build_vector_type (cmp_type, TYPE_VECTOR_SUBPARTS (mask_type));
1224 33157 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1225 : {
1226 11526 : rgc.compare_type = cmp_vectype;
1227 11526 : break;
1228 : }
1229 : }
1230 : }
1231 17473 : if (!rgc.compare_type)
1232 : {
1233 : ok = false;
1234 : break;
1235 : }
1236 : }
1237 23482 : if (!ok)
1238 : {
1239 6092 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
1240 6092 : return false;
1241 : }
1242 :
1243 17390 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = error_mark_node;
1244 17390 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1245 17390 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_avx512;
1246 17390 : return true;
1247 23482 : }
1248 :
1249 : /* Check whether we can use vector access with length based on precision
1250 : comparison. So far, to keep it simple, we only allow the case that the
1251 : precision of the target supported length is larger than the precision
1252 : required by loop niters. */
1253 :
1254 : static bool
1255 6 : vect_verify_loop_lens (loop_vec_info loop_vinfo)
1256 : {
1257 6 : if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
1258 : return false;
1259 :
1260 0 : if (!VECTOR_MODE_P (loop_vinfo->vector_mode))
1261 : return false;
1262 :
1263 0 : machine_mode len_load_mode, len_store_mode;
1264 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
1265 0 : .exists (&len_load_mode))
1266 0 : return false;
1267 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
1268 0 : .exists (&len_store_mode))
1269 0 : return false;
1270 :
1271 0 : signed char partial_load_bias = internal_len_load_store_bias
1272 0 : (IFN_LEN_LOAD, len_load_mode);
1273 :
1274 0 : signed char partial_store_bias = internal_len_load_store_bias
1275 0 : (IFN_LEN_STORE, len_store_mode);
1276 :
1277 0 : gcc_assert (partial_load_bias == partial_store_bias);
1278 :
1279 0 : if (partial_load_bias == VECT_PARTIAL_BIAS_UNSUPPORTED)
1280 : return false;
1281 :
1282 : /* If the backend requires a bias of -1 for LEN_LOAD, we must not emit
1283 : len_loads with a length of zero. In order to avoid that we prohibit
1284 : more than one loop length here. */
1285 0 : if (partial_load_bias == -1
1286 0 : && LOOP_VINFO_LENS (loop_vinfo).length () > 1)
1287 : return false;
1288 :
1289 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) = partial_load_bias;
1290 :
1291 0 : unsigned int max_nitems_per_iter = 1;
1292 0 : unsigned int i;
1293 0 : rgroup_controls *rgl;
1294 : /* Find the maximum number of items per iteration for every rgroup. */
1295 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), i, rgl)
1296 : {
1297 0 : unsigned nitems_per_iter = rgl->max_nscalars_per_iter * rgl->factor;
1298 0 : max_nitems_per_iter = MAX (max_nitems_per_iter, nitems_per_iter);
1299 : }
1300 :
1301 : /* Work out how many bits we need to represent the length limit. */
1302 0 : unsigned int min_ni_prec
1303 0 : = vect_min_prec_for_max_niters (loop_vinfo, max_nitems_per_iter);
1304 :
1305 : /* Now use the maximum of below precisions for one suitable IV type:
1306 : - the IV's natural precision
1307 : - the precision needed to hold: the maximum number of scalar
1308 : iterations multiplied by the scale factor (min_ni_prec above)
1309 : - the Pmode precision
1310 :
1311 : If min_ni_prec is less than the precision of the current niters,
1312 : we prefer to still use the niters type. Prefer to use Pmode and
1313 : wider IV to avoid narrow conversions. */
1314 :
1315 0 : unsigned int ni_prec
1316 0 : = TYPE_PRECISION (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)));
1317 0 : min_ni_prec = MAX (min_ni_prec, ni_prec);
1318 0 : min_ni_prec = MAX (min_ni_prec, GET_MODE_BITSIZE (Pmode));
1319 :
1320 0 : tree iv_type = NULL_TREE;
1321 0 : opt_scalar_int_mode tmode_iter;
1322 0 : FOR_EACH_MODE_IN_CLASS (tmode_iter, MODE_INT)
1323 : {
1324 0 : scalar_mode tmode = tmode_iter.require ();
1325 0 : unsigned int tbits = GET_MODE_BITSIZE (tmode);
1326 :
1327 : /* ??? Do we really want to construct one IV whose precision exceeds
1328 : BITS_PER_WORD? */
1329 0 : if (tbits > BITS_PER_WORD)
1330 : break;
1331 :
1332 : /* Find the first available standard integral type. */
1333 0 : if (tbits >= min_ni_prec && targetm.scalar_mode_supported_p (tmode))
1334 : {
1335 0 : iv_type = build_nonstandard_integer_type (tbits, true);
1336 0 : break;
1337 : }
1338 : }
1339 :
1340 0 : if (!iv_type)
1341 : {
1342 0 : if (dump_enabled_p ())
1343 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1344 : "can't vectorize with length-based partial vectors"
1345 : " because there is no suitable iv type.\n");
1346 : return false;
1347 : }
1348 :
1349 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = iv_type;
1350 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1351 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_len;
1352 :
1353 0 : return true;
1354 : }
1355 :
1356 : /* Calculate the cost of one scalar iteration of the loop. */
1357 : static void
1358 373736 : vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
1359 : {
1360 373736 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1361 373736 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
1362 373736 : int nbbs = loop->num_nodes, factor;
1363 373736 : int innerloop_iters, i;
1364 :
1365 373736 : DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
1366 :
1367 : /* Gather costs for statements in the scalar loop. */
1368 :
1369 : /* FORNOW. */
1370 373736 : innerloop_iters = 1;
1371 373736 : if (loop->inner)
1372 1624 : innerloop_iters = LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
1373 :
1374 1287032 : for (i = 0; i < nbbs; i++)
1375 : {
1376 913296 : gimple_stmt_iterator si;
1377 913296 : basic_block bb = bbs[i];
1378 :
1379 913296 : if (bb->loop_father == loop->inner)
1380 : factor = innerloop_iters;
1381 : else
1382 910048 : factor = 1;
1383 :
1384 7552702 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
1385 : {
1386 5726110 : gimple *stmt = gsi_stmt (si);
1387 5726110 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
1388 :
1389 5726110 : if (!is_gimple_assign (stmt)
1390 : && !is_gimple_call (stmt)
1391 : && !is_a<gcond *> (stmt))
1392 2112662 : continue;
1393 :
1394 : /* Skip stmts that are not vectorized inside the loop. */
1395 3613448 : stmt_vec_info vstmt_info = vect_stmt_to_vectorize (stmt_info);
1396 3613448 : if (!STMT_VINFO_RELEVANT_P (vstmt_info)
1397 1785991 : && (!STMT_VINFO_LIVE_P (vstmt_info)
1398 53 : || !VECTORIZABLE_CYCLE_DEF
1399 : (STMT_VINFO_DEF_TYPE (vstmt_info))))
1400 1785991 : continue;
1401 :
1402 1827457 : vect_cost_for_stmt kind;
1403 1827457 : if (STMT_VINFO_DATA_REF (stmt_info))
1404 : {
1405 881679 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
1406 : kind = scalar_load;
1407 : else
1408 328310 : kind = scalar_store;
1409 : }
1410 945778 : else if (vect_nop_conversion_p (stmt_info))
1411 53882 : continue;
1412 : else
1413 : kind = scalar_stmt;
1414 :
1415 : /* We are using vect_prologue here to avoid scaling twice
1416 : by the inner loop factor. */
1417 1773575 : record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
1418 : factor, kind, stmt_info, 0, vect_body);
1419 : }
1420 : }
1421 :
1422 : /* Now accumulate cost. */
1423 373736 : loop_vinfo->scalar_costs = init_cost (loop_vinfo, true);
1424 373736 : add_stmt_costs (loop_vinfo->scalar_costs,
1425 : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo));
1426 373736 : loop_vinfo->scalar_costs->finish_cost (nullptr);
1427 373736 : }
1428 :
1429 : /* Function vect_analyze_loop_form.
1430 :
1431 : Verify that certain CFG restrictions hold, including:
1432 : - the loop has a pre-header
1433 : - the loop has a single entry
1434 : - nested loops can have only a single exit.
1435 : - the loop exit condition is simple enough
1436 : - the number of iterations can be analyzed, i.e, a countable loop. The
1437 : niter could be analyzed under some assumptions. */
1438 :
1439 : opt_result
1440 464815 : vect_analyze_loop_form (class loop *loop, gimple *loop_vectorized_call,
1441 : vect_loop_form_info *info)
1442 : {
1443 464815 : DUMP_VECT_SCOPE ("vect_analyze_loop_form");
1444 :
1445 464815 : edge exit_e = vec_init_loop_exit_info (loop);
1446 464815 : if (!exit_e)
1447 30380 : return opt_result::failure_at (vect_location,
1448 : "not vectorized:"
1449 : " Infinite loop detected.\n");
1450 434435 : if (loop_vectorized_call)
1451 : {
1452 28903 : tree arg = gimple_call_arg (loop_vectorized_call, 1);
1453 28903 : class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
1454 28903 : edge scalar_exit_e = vec_init_loop_exit_info (scalar_loop);
1455 28903 : if (!scalar_exit_e)
1456 0 : return opt_result::failure_at (vect_location,
1457 : "not vectorized:"
1458 : " could not determine main exit from"
1459 : " loop with multiple exits.\n");
1460 : }
1461 :
1462 434435 : info->loop_exit = exit_e;
1463 434435 : if (dump_enabled_p ())
1464 16095 : dump_printf_loc (MSG_NOTE, vect_location,
1465 : "using as main loop exit: %d -> %d [AUX: %p]\n",
1466 16095 : exit_e->src->index, exit_e->dest->index, exit_e->aux);
1467 :
1468 : /* Check if we have any control flow that doesn't leave the loop. */
1469 434435 : basic_block *bbs = get_loop_body (loop);
1470 1857563 : for (unsigned i = 0; i < loop->num_nodes; i++)
1471 1107011 : if (EDGE_COUNT (bbs[i]->succs) != 1
1472 1107011 : && (EDGE_COUNT (bbs[i]->succs) != 2
1473 663133 : || !loop_exits_from_bb_p (bbs[i]->loop_father, bbs[i])))
1474 : {
1475 118318 : free (bbs);
1476 118318 : return opt_result::failure_at (vect_location,
1477 : "not vectorized:"
1478 : " unsupported control flow in loop.\n");
1479 : }
1480 :
1481 : /* Check if we have any control flow that doesn't leave the loop. */
1482 317221 : bool has_phi = false;
1483 317221 : for (unsigned i = 0; i < loop->num_nodes; i++)
1484 316761 : if (!gimple_seq_empty_p (phi_nodes (bbs[i])))
1485 : {
1486 : has_phi = true;
1487 : break;
1488 : }
1489 316117 : if (!has_phi)
1490 460 : return opt_result::failure_at (vect_location,
1491 : "not vectorized:"
1492 : " no scalar evolution detected in loop.\n");
1493 :
1494 315657 : free (bbs);
1495 :
1496 : /* Different restrictions apply when we are considering an inner-most loop,
1497 : vs. an outer (nested) loop.
1498 : (FORNOW. May want to relax some of these restrictions in the future). */
1499 :
1500 315657 : info->inner_loop_cond = NULL;
1501 315657 : if (!loop->inner)
1502 : {
1503 : /* Inner-most loop. */
1504 :
1505 296894 : if (empty_block_p (loop->header))
1506 0 : return opt_result::failure_at (vect_location,
1507 : "not vectorized: empty loop.\n");
1508 : }
1509 : else
1510 : {
1511 18763 : class loop *innerloop = loop->inner;
1512 18763 : edge entryedge;
1513 :
1514 : /* Nested loop. We currently require that the loop is doubly-nested,
1515 : contains a single inner loop with a single exit to the block
1516 : with the single exit condition in the outer loop.
1517 : Vectorizable outer-loops look like this:
1518 :
1519 : (pre-header)
1520 : |
1521 : header <---+
1522 : | |
1523 : inner-loop |
1524 : | |
1525 : tail ------+
1526 : |
1527 : (exit-bb)
1528 :
1529 : The inner-loop also has the properties expected of inner-most loops
1530 : as described above. */
1531 :
1532 18763 : if ((loop->inner)->inner || (loop->inner)->next)
1533 3023 : return opt_result::failure_at (vect_location,
1534 : "not vectorized:"
1535 : " multiple nested loops.\n");
1536 :
1537 15740 : entryedge = loop_preheader_edge (innerloop);
1538 15740 : if (entryedge->src != loop->header
1539 15198 : || !single_exit (innerloop)
1540 27241 : || single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
1541 4543 : return opt_result::failure_at (vect_location,
1542 : "not vectorized:"
1543 : " unsupported outerloop form.\n");
1544 :
1545 : /* Analyze the inner-loop. */
1546 11197 : vect_loop_form_info inner;
1547 11197 : opt_result res = vect_analyze_loop_form (loop->inner, NULL, &inner);
1548 11197 : if (!res)
1549 : {
1550 417 : if (dump_enabled_p ())
1551 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1552 : "not vectorized: Bad inner loop.\n");
1553 417 : return res;
1554 : }
1555 :
1556 : /* Don't support analyzing niter under assumptions for inner
1557 : loop. */
1558 10780 : if (!integer_onep (inner.assumptions))
1559 263 : return opt_result::failure_at (vect_location,
1560 : "not vectorized: Bad inner loop.\n");
1561 :
1562 10517 : if (inner.number_of_iterations == chrec_dont_know
1563 10517 : || !expr_invariant_in_loop_p (loop, inner.number_of_iterations))
1564 1842 : return opt_result::failure_at (vect_location,
1565 : "not vectorized: inner-loop count not"
1566 : " invariant.\n");
1567 :
1568 8675 : if (dump_enabled_p ())
1569 1050 : dump_printf_loc (MSG_NOTE, vect_location,
1570 : "Considering outer-loop vectorization.\n");
1571 8675 : info->inner_loop_cond = inner.conds[0];
1572 11197 : }
1573 :
1574 305569 : if (EDGE_COUNT (loop->header->preds) != 2)
1575 0 : return opt_result::failure_at (vect_location,
1576 : "not vectorized:"
1577 : " too many incoming edges.\n");
1578 :
1579 : /* We assume that the latch is empty. */
1580 305569 : basic_block latch = loop->latch;
1581 305569 : do
1582 : {
1583 305569 : if (!empty_block_p (latch)
1584 305569 : || !gimple_seq_empty_p (phi_nodes (latch)))
1585 20885 : return opt_result::failure_at (vect_location,
1586 : "not vectorized: latch block not "
1587 : "empty.\n");
1588 284684 : latch = single_pred (latch);
1589 : }
1590 569368 : while (single_succ_p (latch));
1591 :
1592 : /* Make sure there is no abnormal exit. */
1593 284684 : auto_vec<edge> exits = get_loop_exit_edges (loop);
1594 979351 : for (edge e : exits)
1595 : {
1596 410016 : if (e->flags & EDGE_ABNORMAL)
1597 33 : return opt_result::failure_at (vect_location,
1598 : "not vectorized:"
1599 : " abnormal loop exit edge.\n");
1600 : }
1601 :
1602 284651 : info->conds
1603 284651 : = vect_get_loop_niters (loop, exit_e, &info->assumptions,
1604 : &info->number_of_iterations,
1605 284651 : &info->number_of_iterationsm1);
1606 284651 : if (info->conds.is_empty ())
1607 34 : return opt_result::failure_at
1608 34 : (vect_location,
1609 : "not vectorized: complicated exit condition.\n");
1610 :
1611 : /* Determine what the primary and alternate exit conds are. */
1612 694566 : for (unsigned i = 0; i < info->conds.length (); i++)
1613 : {
1614 409949 : gcond *cond = info->conds[i];
1615 409949 : if (exit_e->src == gimple_bb (cond))
1616 284617 : std::swap (info->conds[0], info->conds[i]);
1617 : }
1618 :
1619 284617 : if (chrec_contains_undetermined (info->number_of_iterations))
1620 : {
1621 59909 : if (dump_enabled_p ())
1622 263 : dump_printf_loc (MSG_NOTE, vect_location,
1623 : "Loop being analyzed as uncounted.\n");
1624 59909 : if (loop->inner)
1625 565 : return opt_result::failure_at
1626 565 : (vect_location,
1627 : "not vectorized: outer loop vectorization of uncounted loops"
1628 : " is unsupported.\n");
1629 59344 : return opt_result::success ();
1630 : }
1631 :
1632 224708 : if (integer_zerop (info->assumptions))
1633 4 : return opt_result::failure_at
1634 4 : (info->conds[0],
1635 : "not vectorized: number of iterations cannot be computed.\n");
1636 :
1637 224704 : if (integer_zerop (info->number_of_iterations))
1638 12 : return opt_result::failure_at
1639 12 : (info->conds[0],
1640 : "not vectorized: number of iterations = 0.\n");
1641 :
1642 224692 : if (!(tree_fits_shwi_p (info->number_of_iterations)
1643 124199 : && tree_to_shwi (info->number_of_iterations) > 0))
1644 : {
1645 100493 : if (dump_enabled_p ())
1646 : {
1647 2504 : dump_printf_loc (MSG_NOTE, vect_location,
1648 : "Symbolic number of iterations is ");
1649 2504 : dump_generic_expr (MSG_NOTE, TDF_DETAILS, info->number_of_iterations);
1650 2504 : dump_printf (MSG_NOTE, "\n");
1651 : }
1652 : }
1653 :
1654 224692 : if (!integer_onep (info->assumptions))
1655 : {
1656 8816 : if (dump_enabled_p ())
1657 : {
1658 75 : dump_printf_loc (MSG_NOTE, vect_location,
1659 : "Loop to be versioned with niter assumption ");
1660 75 : dump_generic_expr (MSG_NOTE, TDF_SLIM, info->assumptions);
1661 75 : dump_printf (MSG_NOTE, "\n");
1662 : }
1663 : }
1664 :
1665 224692 : return opt_result::success ();
1666 284684 : }
1667 :
1668 : /* Create a loop_vec_info for LOOP with SHARED and the
1669 : vect_analyze_loop_form result. */
1670 :
1671 : loop_vec_info
1672 589396 : vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
1673 : const vect_loop_form_info *info,
1674 : loop_vec_info orig_loop_info)
1675 : {
1676 589396 : loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared);
1677 589396 : LOOP_VINFO_NITERSM1 (loop_vinfo) = info->number_of_iterationsm1;
1678 589396 : LOOP_VINFO_NITERS (loop_vinfo) = info->number_of_iterations;
1679 589396 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = info->number_of_iterations;
1680 589396 : LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = orig_loop_info;
1681 589396 : if (orig_loop_info && LOOP_VINFO_EPILOGUE_P (orig_loop_info))
1682 340 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo)
1683 340 : = LOOP_VINFO_MAIN_LOOP_INFO (orig_loop_info);
1684 : else
1685 589056 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) = orig_loop_info;
1686 : /* Also record the assumptions for versioning. */
1687 589396 : if (!integer_onep (info->assumptions) && !orig_loop_info)
1688 19965 : LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = info->assumptions;
1689 :
1690 2618132 : for (gcond *cond : info->conds)
1691 : {
1692 849944 : stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
1693 : /* Mark the statement as a condition. */
1694 849944 : STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
1695 : }
1696 :
1697 589396 : unsigned cond_id = 0;
1698 589396 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
1699 502678 : LOOP_VINFO_LOOP_IV_COND (loop_vinfo) = info->conds[cond_id++];
1700 :
1701 936662 : for (; cond_id < info->conds.length (); cond_id ++)
1702 347266 : LOOP_VINFO_LOOP_CONDS (loop_vinfo).safe_push (info->conds[cond_id]);
1703 :
1704 589396 : LOOP_VINFO_MAIN_EXIT (loop_vinfo) = info->loop_exit;
1705 :
1706 : /* Check to see if we're vectorizing multiple exits. */
1707 589396 : LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
1708 589396 : = !LOOP_VINFO_LOOP_CONDS (loop_vinfo).is_empty ();
1709 :
1710 : /* At the moment we can't support no epilogs for multiple exits, result of
1711 : the first compare should be masked by that of the second. We can only
1712 : allow it if the early exits have the same live values. for differing
1713 : values we have to calculate a third mask to disambiguate. */
1714 589396 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo)
1715 589396 : = LOOP_VINFO_LOOP_CONDS (loop_vinfo).length () > 1;
1716 :
1717 589396 : if (info->inner_loop_cond)
1718 : {
1719 : /* If we have an estimate on the number of iterations of the inner
1720 : loop use that to limit the scale for costing, otherwise use
1721 : --param vect-inner-loop-cost-factor literally. */
1722 9098 : widest_int nit;
1723 9098 : if (estimated_stmt_executions (loop->inner, &nit))
1724 7785 : LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo)
1725 7785 : = wi::smin (nit, param_vect_inner_loop_cost_factor).to_uhwi ();
1726 9098 : }
1727 :
1728 589396 : return loop_vinfo;
1729 : }
1730 :
1731 :
1732 :
1733 : /* Return true if we know that the iteration count is smaller than the
1734 : vectorization factor. Return false if it isn't, or if we can't be sure
1735 : either way. */
1736 :
1737 : static bool
1738 155641 : vect_known_niters_smaller_than_vf (loop_vec_info loop_vinfo)
1739 : {
1740 155641 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1741 :
1742 155641 : HOST_WIDE_INT max_niter;
1743 155641 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1744 80015 : max_niter = LOOP_VINFO_INT_NITERS (loop_vinfo);
1745 : else
1746 75626 : max_niter = max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo));
1747 :
1748 155641 : if (max_niter != -1 && (unsigned HOST_WIDE_INT) max_niter < assumed_vf)
1749 11056 : return true;
1750 :
1751 : return false;
1752 : }
1753 :
1754 : /* Analyze the cost of the loop described by LOOP_VINFO. Decide if it
1755 : is worthwhile to vectorize. Return 1 if definitely yes, 0 if
1756 : definitely no, or -1 if it's worth retrying. */
1757 :
1758 : static int
1759 155654 : vect_analyze_loop_costing (loop_vec_info loop_vinfo,
1760 : unsigned *suggested_unroll_factor)
1761 : {
1762 155654 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1763 155654 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1764 :
1765 : /* Only loops that can handle partially-populated vectors can have iteration
1766 : counts less than the vectorization factor. */
1767 155654 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
1768 155654 : && vect_known_niters_smaller_than_vf (loop_vinfo))
1769 : {
1770 11046 : if (dump_enabled_p ())
1771 236 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1772 : "not vectorized: iteration count smaller than "
1773 : "vectorization factor.\n");
1774 : return 0;
1775 : }
1776 :
1777 : /* If we know the number of iterations we can do better, for the
1778 : epilogue we can also decide whether the main loop leaves us
1779 : with enough iterations, preferring a smaller vector epilog then
1780 : also possibly used for the case we skip the vector loop. */
1781 144608 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1782 : {
1783 70249 : widest_int scalar_niters
1784 70249 : = wi::to_widest (LOOP_VINFO_NITERSM1 (loop_vinfo)) + 1;
1785 70249 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1786 : {
1787 2647 : loop_vec_info orig_loop_vinfo
1788 : = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
1789 2647 : loop_vec_info main_loop_vinfo
1790 : = LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo);
1791 2647 : unsigned lowest_vf
1792 2647 : = constant_lower_bound (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo));
1793 2647 : int prolog_peeling = 0;
1794 2647 : if (!vect_use_loop_mask_for_alignment_p (main_loop_vinfo))
1795 2647 : prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
1796 2647 : if (prolog_peeling >= 0
1797 2647 : && known_eq (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
1798 : lowest_vf))
1799 : {
1800 5284 : unsigned gap
1801 2642 : = LOOP_VINFO_PEELING_FOR_GAPS (main_loop_vinfo) ? 1 : 0;
1802 5284 : scalar_niters = ((scalar_niters - gap - prolog_peeling)
1803 5284 : % lowest_vf + gap);
1804 : }
1805 : }
1806 : /* Reject vectorizing for a single scalar iteration, even if
1807 : we could in principle implement that using partial vectors.
1808 : But allow such vectorization if VF == 1 in case we do not
1809 : need to peel for gaps (if we need, avoid vectorization for
1810 : reasons of code footprint). */
1811 70249 : unsigned peeling_gap = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo);
1812 70249 : if (scalar_niters <= peeling_gap + 1
1813 70249 : && (assumed_vf > 1 || peeling_gap != 0))
1814 : {
1815 662 : if (dump_enabled_p ())
1816 162 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1817 : "not vectorized: loop only has a single "
1818 : "scalar iteration.\n");
1819 : return 0;
1820 : }
1821 :
1822 69587 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
1823 : {
1824 : /* Check that the loop processes at least one full vector. */
1825 69574 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1826 69574 : if (known_lt (scalar_niters, vf))
1827 : {
1828 350 : if (dump_enabled_p ())
1829 296 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1830 : "loop does not have enough iterations "
1831 : "to support vectorization.\n");
1832 396 : return 0;
1833 : }
1834 :
1835 : /* If we need to peel an extra epilogue iteration to handle data
1836 : accesses with gaps, check that there are enough scalar iterations
1837 : available.
1838 :
1839 : The check above is redundant with this one when peeling for gaps,
1840 : but the distinction is useful for diagnostics. */
1841 69224 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
1842 69539 : && known_le (scalar_niters, vf))
1843 : {
1844 46 : if (dump_enabled_p ())
1845 9 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1846 : "loop does not have enough iterations "
1847 : "to support peeling for gaps.\n");
1848 : return 0;
1849 : }
1850 : }
1851 70249 : }
1852 :
1853 : /* If using the "very cheap" model. reject cases in which we'd keep
1854 : a copy of the scalar code (even if we might be able to vectorize it). */
1855 143550 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1856 143550 : && (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1857 76200 : || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
1858 : {
1859 721 : if (dump_enabled_p ())
1860 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1861 : "some scalar iterations would need to be peeled\n");
1862 : return 0;
1863 : }
1864 :
1865 142829 : int min_profitable_iters, min_profitable_estimate;
1866 142829 : vect_estimate_min_profitable_iters (loop_vinfo, &min_profitable_iters,
1867 : &min_profitable_estimate,
1868 : suggested_unroll_factor);
1869 :
1870 142829 : if (min_profitable_iters < 0)
1871 : {
1872 24415 : if (dump_enabled_p ())
1873 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1874 : "not vectorized: vectorization not profitable.\n");
1875 24415 : if (dump_enabled_p ())
1876 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1877 : "not vectorized: vector version will never be "
1878 : "profitable.\n");
1879 : return -1;
1880 : }
1881 :
1882 118414 : int min_scalar_loop_bound = (param_min_vect_loop_bound
1883 118414 : * assumed_vf);
1884 :
1885 : /* Use the cost model only if it is more conservative than user specified
1886 : threshold. */
1887 118414 : unsigned int th = (unsigned) MAX (min_scalar_loop_bound,
1888 : min_profitable_iters);
1889 :
1890 118414 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = th;
1891 :
1892 63871 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1893 182285 : && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
1894 : {
1895 453 : if (dump_enabled_p ())
1896 1 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1897 : "not vectorized: vectorization not profitable.\n");
1898 453 : if (dump_enabled_p ())
1899 1 : dump_printf_loc (MSG_NOTE, vect_location,
1900 : "not vectorized: iteration count smaller than user "
1901 : "specified loop bound parameter or minimum profitable "
1902 : "iterations (whichever is more conservative).\n");
1903 : return 0;
1904 : }
1905 :
1906 : /* The static profitablity threshold min_profitable_estimate includes
1907 : the cost of having to check at runtime whether the scalar loop
1908 : should be used instead. If it turns out that we don't need or want
1909 : such a check, the threshold we should use for the static estimate
1910 : is simply the point at which the vector loop becomes more profitable
1911 : than the scalar loop. */
1912 117961 : if (min_profitable_estimate > min_profitable_iters
1913 25144 : && !LOOP_REQUIRES_VERSIONING (loop_vinfo)
1914 24577 : && !LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
1915 617 : && !LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1916 118578 : && !vect_apply_runtime_profitability_check_p (loop_vinfo))
1917 : {
1918 14 : if (dump_enabled_p ())
1919 8 : dump_printf_loc (MSG_NOTE, vect_location, "no need for a runtime"
1920 : " choice between the scalar and vector loops\n");
1921 14 : min_profitable_estimate = min_profitable_iters;
1922 : }
1923 :
1924 : /* If the vector loop needs multiple iterations to be beneficial then
1925 : things are probably too close to call, and the conservative thing
1926 : would be to stick with the scalar code. */
1927 117961 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1928 117961 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1929 : {
1930 18477 : if (dump_enabled_p ())
1931 225 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1932 : "one iteration of the vector loop would be"
1933 : " more expensive than the equivalent number of"
1934 : " iterations of the scalar loop\n");
1935 : return 0;
1936 : }
1937 :
1938 99484 : HOST_WIDE_INT estimated_niter;
1939 :
1940 : /* If we are vectorizing an epilogue then we know the maximum number of
1941 : scalar iterations it will cover is at least one lower than the
1942 : vectorization factor of the main loop. */
1943 99484 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1944 11985 : estimated_niter
1945 11985 : = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
1946 : else
1947 : {
1948 87499 : estimated_niter = estimated_stmt_executions_int (loop);
1949 87499 : if (estimated_niter == -1)
1950 31696 : estimated_niter = likely_max_stmt_executions_int (loop);
1951 : }
1952 43681 : if (estimated_niter != -1
1953 96532 : && ((unsigned HOST_WIDE_INT) estimated_niter
1954 96532 : < MAX (th, (unsigned) min_profitable_estimate)))
1955 : {
1956 4245 : if (dump_enabled_p ())
1957 34 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1958 : "not vectorized: estimated iteration count too "
1959 : "small.\n");
1960 4245 : if (dump_enabled_p ())
1961 34 : dump_printf_loc (MSG_NOTE, vect_location,
1962 : "not vectorized: estimated iteration count smaller "
1963 : "than specified loop bound parameter or minimum "
1964 : "profitable iterations (whichever is more "
1965 : "conservative).\n");
1966 : return -1;
1967 : }
1968 :
1969 : /* As we cannot use a runtime check to gate profitability for uncounted
1970 : loops require either an estimate or if none, at least a profitable
1971 : vectorization within the first vector iteration (that condition
1972 : will practically never be true due to the required epilog and
1973 : likely alignment prologue). */
1974 95239 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
1975 163 : && estimated_niter == -1
1976 95375 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1977 : {
1978 120 : if (dump_enabled_p ())
1979 2 : dump_printf_loc (MSG_NOTE, vect_location,
1980 : "not vectorized: no loop iteration estimate on the "
1981 : "uncounted loop and not trivially profitable.\n");
1982 : return -1;
1983 : }
1984 :
1985 : return 1;
1986 : }
1987 :
1988 : /* Gather data references in LOOP with body BBS and store them into
1989 : *DATAREFS. */
1990 :
1991 : static opt_result
1992 282215 : vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
1993 : vec<data_reference_p> *datarefs)
1994 : {
1995 845119 : for (unsigned i = 0; i < loop->num_nodes; i++)
1996 1251858 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
1997 5449802 : !gsi_end_p (gsi); gsi_next (&gsi))
1998 : {
1999 4886898 : gimple *stmt = gsi_stmt (gsi);
2000 4886898 : if (is_gimple_debug (stmt))
2001 2346449 : continue;
2002 2540579 : opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs,
2003 : NULL, 0);
2004 2540579 : if (!res)
2005 : {
2006 63155 : if (is_gimple_call (stmt) && loop->safelen)
2007 : {
2008 406 : tree fndecl = gimple_call_fndecl (stmt), op;
2009 406 : if (fndecl == NULL_TREE
2010 406 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
2011 : {
2012 0 : fndecl = gimple_call_arg (stmt, 0);
2013 0 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
2014 0 : fndecl = TREE_OPERAND (fndecl, 0);
2015 0 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
2016 : }
2017 406 : if (fndecl != NULL_TREE)
2018 : {
2019 369 : cgraph_node *node = cgraph_node::get (fndecl);
2020 369 : if (node != NULL && node->simd_clones != NULL)
2021 : {
2022 131 : unsigned int j, n = gimple_call_num_args (stmt);
2023 545 : for (j = 0; j < n; j++)
2024 : {
2025 284 : op = gimple_call_arg (stmt, j);
2026 284 : if (DECL_P (op)
2027 284 : || (REFERENCE_CLASS_P (op)
2028 0 : && get_base_address (op)))
2029 : break;
2030 : }
2031 131 : op = gimple_call_lhs (stmt);
2032 : /* Ignore #pragma omp declare simd functions
2033 : if they don't have data references in the
2034 : call stmt itself. */
2035 261 : if (j == n
2036 131 : && !(op
2037 120 : && (DECL_P (op)
2038 120 : || (REFERENCE_CLASS_P (op)
2039 0 : && get_base_address (op)))))
2040 130 : continue;
2041 : }
2042 : }
2043 : }
2044 63025 : return res;
2045 : }
2046 : /* If dependence analysis will give up due to the limit on the
2047 : number of datarefs stop here and fail fatally. */
2048 4343005 : if (datarefs->length ()
2049 1865581 : > (unsigned)param_loop_max_datarefs_for_datadeps)
2050 0 : return opt_result::failure_at (stmt, "exceeded param "
2051 : "loop-max-datarefs-for-datadeps\n");
2052 : }
2053 219190 : return opt_result::success ();
2054 : }
2055 :
2056 : /* Determine if operating on full vectors for LOOP_VINFO might leave
2057 : some scalar iterations still to do. If so, decide how we should
2058 : handle those scalar iterations. The possibilities are:
2059 :
2060 : (1) Make LOOP_VINFO operate on partial vectors instead of full vectors.
2061 : In this case:
2062 :
2063 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == true
2064 : LOOP_VINFO_PEELING_FOR_NITER == false
2065 :
2066 : (2) Make LOOP_VINFO operate on full vectors and use an epilogue loop
2067 : to handle the remaining scalar iterations. In this case:
2068 :
2069 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == false
2070 : LOOP_VINFO_PEELING_FOR_NITER == true
2071 :
2072 : The MASKED_P argument specifies to what extent
2073 : param_vect_partial_vector_usage is to be honored. For MASKED_P == 0
2074 : no partial vectors are to be used, for MASKED_P == -1 it's
2075 : param_vect_partial_vector_usage that gets to decide whether we may
2076 : consider partial vector usage. For MASKED_P == 1 partial vectors
2077 : may be used if possible.
2078 :
2079 : */
2080 :
2081 : static opt_result
2082 156564 : vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
2083 : int masked_p)
2084 : {
2085 : /* Determine whether there would be any scalar iterations left over. */
2086 156564 : bool need_peeling_or_partial_vectors_p
2087 156564 : = vect_need_peeling_or_partial_vectors_p (loop_vinfo);
2088 :
2089 : /* Decide whether to vectorize the loop with partial vectors. */
2090 156564 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2091 156564 : if (masked_p == 0
2092 156564 : || (masked_p == -1 && param_vect_partial_vector_usage == 0))
2093 : /* If requested explicitly do not use partial vectors. */
2094 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2095 211 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2096 69 : && LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo))
2097 0 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2098 211 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2099 69 : && need_peeling_or_partial_vectors_p)
2100 : {
2101 : /* For partial-vector-usage=1, try to push the handling of partial
2102 : vectors to the epilogue, with the main loop continuing to operate
2103 : on full vectors.
2104 :
2105 : If we are unrolling we also do not want to use partial vectors. This
2106 : is to avoid the overhead of generating multiple masks and also to
2107 : avoid having to execute entire iterations of FALSE masked instructions
2108 : when dealing with one or less full iterations.
2109 :
2110 : ??? We could then end up failing to use partial vectors if we
2111 : decide to peel iterations into a prologue, and if the main loop
2112 : then ends up processing fewer than VF iterations. */
2113 47 : if ((param_vect_partial_vector_usage == 1
2114 14 : || loop_vinfo->suggested_unroll_factor > 1)
2115 33 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2116 69 : && !vect_known_niters_smaller_than_vf (loop_vinfo))
2117 : ;
2118 : else
2119 35 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2120 : }
2121 :
2122 156564 : if (LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo)
2123 0 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2124 0 : return opt_result::failure_at (vect_location,
2125 : "not vectorized: loop needs but cannot "
2126 : "use partial vectors\n");
2127 :
2128 156564 : if (dump_enabled_p ())
2129 12585 : dump_printf_loc (MSG_NOTE, vect_location,
2130 : "operating on %s vectors%s.\n",
2131 12585 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2132 : ? "partial" : "full",
2133 12585 : LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2134 : ? " for epilogue loop" : "");
2135 :
2136 156564 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2137 313128 : = (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2138 156564 : && need_peeling_or_partial_vectors_p);
2139 :
2140 156564 : return opt_result::success ();
2141 : }
2142 :
2143 : /* Function vect_analyze_loop_2.
2144 :
2145 : Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
2146 : analyses will record information in some members of LOOP_VINFO. FATAL
2147 : indicates if some analysis meets fatal error. If one non-NULL pointer
2148 : SUGGESTED_UNROLL_FACTOR is provided, it's intent to be filled with one
2149 : worked out suggested unroll factor, while one NULL pointer shows it's
2150 : going to apply the suggested unroll factor.
2151 : SINGLE_LANE_SLP_DONE_FOR_SUGGESTED_UF is to hold whether single-lane
2152 : slp was forced when the suggested unroll factor was worked out. */
2153 : static opt_result
2154 588697 : vect_analyze_loop_2 (loop_vec_info loop_vinfo, int masked_p, bool &fatal,
2155 : unsigned *suggested_unroll_factor,
2156 : bool& single_lane_slp_done_for_suggested_uf)
2157 : {
2158 588697 : opt_result ok = opt_result::success ();
2159 588697 : int res;
2160 588697 : unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
2161 588697 : loop_vec_info orig_loop_vinfo = NULL;
2162 :
2163 : /* If we are dealing with an epilogue then orig_loop_vinfo points to the
2164 : loop_vec_info of the first vectorized loop. */
2165 588697 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2166 13813 : orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
2167 : else
2168 : orig_loop_vinfo = loop_vinfo;
2169 13813 : gcc_assert (orig_loop_vinfo);
2170 :
2171 : /* We can't mask on niters for uncounted loops due to unknown upper bound. */
2172 588697 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
2173 86718 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2174 :
2175 : /* The first group of checks is independent of the vector size. */
2176 588697 : fatal = true;
2177 :
2178 588697 : if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)
2179 588697 : && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)))
2180 5 : return opt_result::failure_at (vect_location,
2181 : "not vectorized: simd if(0)\n");
2182 :
2183 : /* Find all data references in the loop (which correspond to vdefs/vuses)
2184 : and analyze their evolution in the loop. */
2185 :
2186 588692 : loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
2187 :
2188 : /* Gather the data references. */
2189 588692 : if (!LOOP_VINFO_DATAREFS (loop_vinfo).exists ())
2190 : {
2191 282215 : opt_result res
2192 282215 : = vect_get_datarefs_in_loop (loop, LOOP_VINFO_BBS (loop_vinfo),
2193 : &LOOP_VINFO_DATAREFS (loop_vinfo));
2194 282215 : if (!res)
2195 : {
2196 63025 : if (dump_enabled_p ())
2197 1636 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2198 : "not vectorized: loop contains function "
2199 : "calls or data references that cannot "
2200 : "be analyzed\n");
2201 63025 : return res;
2202 : }
2203 219190 : loop_vinfo->shared->save_datarefs ();
2204 : }
2205 : else
2206 306477 : loop_vinfo->shared->check_datarefs ();
2207 :
2208 : /* Analyze the data references and also adjust the minimal
2209 : vectorization factor according to the loads and stores. */
2210 :
2211 525667 : ok = vect_analyze_data_refs (loop_vinfo, &fatal);
2212 525667 : if (!ok)
2213 : {
2214 73573 : if (dump_enabled_p ())
2215 1241 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2216 : "bad data references.\n");
2217 73573 : return ok;
2218 : }
2219 :
2220 : /* Check if we are applying unroll factor now. */
2221 452094 : bool applying_suggested_uf = loop_vinfo->suggested_unroll_factor > 1;
2222 452094 : gcc_assert (!applying_suggested_uf || !suggested_unroll_factor);
2223 :
2224 : /* When single-lane SLP was forced and we are applying suggested unroll
2225 : factor, keep that decision here. */
2226 904188 : bool force_single_lane = (applying_suggested_uf
2227 452094 : && single_lane_slp_done_for_suggested_uf);
2228 :
2229 : /* Classify all cross-iteration scalar data-flow cycles.
2230 : Cross-iteration cycles caused by virtual phis are analyzed separately. */
2231 452094 : vect_analyze_scalar_cycles (loop_vinfo);
2232 :
2233 452094 : vect_pattern_recog (loop_vinfo);
2234 :
2235 : /* Analyze the access patterns of the data-refs in the loop (consecutive,
2236 : complex, etc.). FORNOW: Only handle consecutive access pattern. */
2237 :
2238 452094 : ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
2239 452094 : if (!ok)
2240 : {
2241 8006 : if (dump_enabled_p ())
2242 292 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2243 : "bad data access.\n");
2244 8006 : return ok;
2245 : }
2246 :
2247 : /* Data-flow analysis to detect stmts that do not need to be vectorized. */
2248 :
2249 444088 : ok = vect_mark_stmts_to_be_vectorized (loop_vinfo, &fatal);
2250 444088 : if (!ok)
2251 : {
2252 45718 : if (dump_enabled_p ())
2253 399 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2254 : "unexpected pattern.\n");
2255 45718 : return ok;
2256 : }
2257 :
2258 : /* While the rest of the analysis below depends on it in some way. */
2259 398370 : fatal = false;
2260 :
2261 : /* Analyze data dependences between the data-refs in the loop
2262 : and adjust the maximum vectorization factor according to
2263 : the dependences.
2264 : FORNOW: fail at the first data dependence that we encounter. */
2265 :
2266 398370 : ok = vect_analyze_data_ref_dependences (loop_vinfo, &max_vf);
2267 398370 : if (!ok)
2268 : {
2269 24634 : if (dump_enabled_p ())
2270 542 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2271 : "bad data dependence.\n");
2272 24634 : return ok;
2273 : }
2274 373736 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
2275 :
2276 : /* Compute the scalar iteration cost. */
2277 373736 : vect_compute_single_scalar_iteration_cost (loop_vinfo);
2278 :
2279 373736 : bool saved_can_use_partial_vectors_p
2280 : = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
2281 :
2282 : /* This is the point where we can re-start analysis with single-lane
2283 : SLP forced. */
2284 518347 : start_over:
2285 :
2286 : /* Check the SLP opportunities in the loop, analyze and build
2287 : SLP trees. */
2288 1036694 : ok = vect_analyze_slp (loop_vinfo, loop_vinfo->stmt_vec_infos.length (),
2289 : force_single_lane);
2290 518347 : if (!ok)
2291 18106 : return ok;
2292 :
2293 : /* If there are any SLP instances mark them as pure_slp and compute
2294 : the overall vectorization factor. */
2295 500241 : if (!vect_make_slp_decision (loop_vinfo))
2296 66517 : return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
2297 :
2298 433724 : if (dump_enabled_p ())
2299 19502 : dump_printf_loc (MSG_NOTE, vect_location, "Loop contains only SLP stmts\n");
2300 :
2301 : /* Dump the vectorization factor from the SLP decision. */
2302 433724 : if (dump_enabled_p ())
2303 : {
2304 19502 : dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
2305 19502 : dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2306 19502 : dump_printf (MSG_NOTE, "\n");
2307 : }
2308 :
2309 : /* We don't expect to have to roll back to anything other than an empty
2310 : set of rgroups. */
2311 433724 : gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
2312 :
2313 : /* Apply the suggested unrolling factor, this was determined by the backend
2314 : during finish_cost the first time we ran the analysis for this
2315 : vector mode. */
2316 433724 : if (applying_suggested_uf)
2317 459 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) *= loop_vinfo->suggested_unroll_factor;
2318 :
2319 : /* Now the vectorization factor is final. */
2320 433724 : poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2321 433724 : gcc_assert (known_ne (vectorization_factor, 0U));
2322 :
2323 : /* Optimize the SLP graph with the vectorization factor fixed. */
2324 433724 : vect_optimize_slp (loop_vinfo);
2325 :
2326 : /* Gather the loads reachable from the SLP graph entries. */
2327 433724 : vect_gather_slp_loads (loop_vinfo);
2328 :
2329 433724 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
2330 : {
2331 14468 : dump_printf_loc (MSG_NOTE, vect_location,
2332 : "vectorization_factor = ");
2333 14468 : dump_dec (MSG_NOTE, vectorization_factor);
2334 14468 : dump_printf (MSG_NOTE, ", niters = %wd\n",
2335 14468 : LOOP_VINFO_INT_NITERS (loop_vinfo));
2336 : }
2337 :
2338 433724 : if (max_vf != MAX_VECTORIZATION_FACTOR
2339 433724 : && maybe_lt (max_vf, LOOP_VINFO_VECT_FACTOR (loop_vinfo)))
2340 41 : return opt_result::failure_at (vect_location, "bad data dependence.\n");
2341 :
2342 433683 : loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
2343 :
2344 : /* Analyze the alignment of the data-refs in the loop. */
2345 433683 : vect_analyze_data_refs_alignment (loop_vinfo);
2346 :
2347 : /* Prune the list of ddrs to be tested at run-time by versioning for alias.
2348 : It is important to call pruning after vect_analyze_data_ref_accesses,
2349 : since we use grouping information gathered by interleaving analysis. */
2350 433683 : ok = vect_prune_runtime_alias_test_list (loop_vinfo);
2351 433683 : if (!ok)
2352 17891 : return ok;
2353 :
2354 : /* Do not invoke vect_enhance_data_refs_alignment for epilogue
2355 : vectorization, since we do not want to add extra peeling or
2356 : add versioning for alignment. */
2357 415792 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2358 : /* This pass will decide on using loop versioning and/or loop peeling in
2359 : order to enhance the alignment of data references in the loop. */
2360 401117 : ok = vect_enhance_data_refs_alignment (loop_vinfo);
2361 415792 : if (!ok)
2362 0 : return ok;
2363 :
2364 : /* Analyze operations in the SLP instances. We can't simply
2365 : remove unsupported SLP instances as this makes the above
2366 : SLP kind detection invalid and might also affect the VF. */
2367 415792 : if (! vect_slp_analyze_operations (loop_vinfo))
2368 : {
2369 259228 : ok = opt_result::failure_at (vect_location,
2370 : "unsupported SLP instances\n");
2371 259228 : goto again;
2372 : }
2373 :
2374 : /* For now, we don't expect to mix both masking and length approaches for one
2375 : loop, disable it if both are recorded. */
2376 156564 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2377 23488 : && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ()
2378 180046 : && !LOOP_VINFO_LENS (loop_vinfo).is_empty ())
2379 : {
2380 0 : if (dump_enabled_p ())
2381 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2382 : "can't vectorize a loop with partial vectors"
2383 : " because we don't expect to mix different"
2384 : " approaches with partial vectors for the"
2385 : " same loop.\n");
2386 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2387 : }
2388 :
2389 : /* If we still have the option of using partial vectors,
2390 : check whether we can generate the necessary loop controls. */
2391 156564 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
2392 : {
2393 23488 : if (!LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
2394 : {
2395 23482 : if (!vect_verify_full_masking (loop_vinfo)
2396 23482 : && !vect_verify_full_masking_avx512 (loop_vinfo))
2397 6092 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2398 : }
2399 : else /* !LOOP_VINFO_LENS (loop_vinfo).is_empty () */
2400 6 : if (!vect_verify_loop_lens (loop_vinfo))
2401 6 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2402 : }
2403 :
2404 : /* Decide whether this loop_vinfo should use partial vectors or peeling,
2405 : assuming that the loop will be used as a main loop. We will redo
2406 : this analysis later if we instead decide to use the loop as an
2407 : epilogue loop. */
2408 156564 : ok = vect_determine_partial_vectors_and_peeling (loop_vinfo, masked_p);
2409 156564 : if (!ok)
2410 0 : return ok;
2411 :
2412 : /* If we're vectorizing a loop that uses length "controls" and
2413 : can iterate more than once, we apply decrementing IV approach
2414 : in loop control. */
2415 156564 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2416 35 : && LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_len
2417 0 : && LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) == 0
2418 156564 : && !(LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2419 0 : && known_le (LOOP_VINFO_INT_NITERS (loop_vinfo),
2420 : LOOP_VINFO_VECT_FACTOR (loop_vinfo))))
2421 0 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = true;
2422 :
2423 : /* If a loop uses length controls and has a decrementing loop control IV,
2424 : we will normally pass that IV through a MIN_EXPR to calcaluate the
2425 : basis for the length controls. E.g. in a loop that processes one
2426 : element per scalar iteration, the number of elements would be
2427 : MIN_EXPR <N, VF>, where N is the number of scalar iterations left.
2428 :
2429 : This MIN_EXPR approach allows us to use pointer IVs with an invariant
2430 : step, since only the final iteration of the vector loop can have
2431 : inactive lanes.
2432 :
2433 : However, some targets have a dedicated instruction for calculating the
2434 : preferred length, given the total number of elements that still need to
2435 : be processed. This is encapsulated in the SELECT_VL internal function.
2436 :
2437 : If the target supports SELECT_VL, we can use it instead of MIN_EXPR
2438 : to determine the basis for the length controls. However, unlike the
2439 : MIN_EXPR calculation, the SELECT_VL calculation can decide to make
2440 : lanes inactive in any iteration of the vector loop, not just the last
2441 : iteration. This SELECT_VL approach therefore requires us to use pointer
2442 : IVs with variable steps.
2443 :
2444 : Once we've decided how many elements should be processed by one
2445 : iteration of the vector loop, we need to populate the rgroup controls.
2446 : If a loop has multiple rgroups, we need to make sure that those rgroups
2447 : "line up" (that is, they must be consistent about which elements are
2448 : active and which aren't). This is done by vect_adjust_loop_lens_control.
2449 :
2450 : In principle, it would be possible to use vect_adjust_loop_lens_control
2451 : on either the result of a MIN_EXPR or the result of a SELECT_VL.
2452 : However:
2453 :
2454 : (1) In practice, it only makes sense to use SELECT_VL when a vector
2455 : operation will be controlled directly by the result. It is not
2456 : worth using SELECT_VL if it would only be the input to other
2457 : calculations.
2458 :
2459 : (2) If we use SELECT_VL for an rgroup that has N controls, each associated
2460 : pointer IV will need N updates by a variable amount (N-1 updates
2461 : within the iteration and 1 update to move to the next iteration).
2462 :
2463 : Because of this, we prefer to use the MIN_EXPR approach whenever there
2464 : is more than one length control.
2465 :
2466 : In addition, SELECT_VL always operates to a granularity of 1 unit.
2467 : If we wanted to use it to control an SLP operation on N consecutive
2468 : elements, we would need to make the SELECT_VL inputs measure scalar
2469 : iterations (rather than elements) and then multiply the SELECT_VL
2470 : result by N. But using SELECT_VL this way is inefficient because
2471 : of (1) above.
2472 :
2473 : 2. We don't apply SELECT_VL on single-rgroup when both (1) and (2) are
2474 : satisfied:
2475 :
2476 : (1). LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) is true.
2477 : (2). LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant () is true.
2478 :
2479 : Since SELECT_VL (variable step) will make SCEV analysis failed and then
2480 : we will fail to gain benefits of following unroll optimizations. We prefer
2481 : using the MIN_EXPR approach in this situation. */
2482 156564 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
2483 : {
2484 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
2485 0 : if (LOOP_VINFO_LENS (loop_vinfo).length () == 1
2486 0 : && LOOP_VINFO_LENS (loop_vinfo)[0].factor == 1
2487 0 : && (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2488 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ()))
2489 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = true;
2490 :
2491 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2492 0 : for (auto rgc : LOOP_VINFO_LENS (loop_vinfo))
2493 0 : if (rgc.type
2494 0 : && !direct_internal_fn_supported_p (IFN_SELECT_VL,
2495 : rgc.type, iv_type,
2496 : OPTIMIZE_FOR_SPEED))
2497 : {
2498 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2499 0 : break;
2500 : }
2501 :
2502 : /* If any of the SLP instances cover more than a single lane
2503 : we cannot use .SELECT_VL at the moment, even if the number
2504 : of lanes is uniform throughout the SLP graph. */
2505 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2506 0 : for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
2507 0 : if (SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)) != 1
2508 0 : && !(SLP_INSTANCE_KIND (inst) == slp_inst_kind_store
2509 0 : && SLP_INSTANCE_TREE (inst)->ldst_lanes))
2510 : {
2511 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2512 0 : break;
2513 : }
2514 : }
2515 :
2516 : /* If we're vectorizing an epilogue loop, the vectorized loop either needs
2517 : to be able to handle fewer than VF scalars, or needs to have a lower VF
2518 : than the main loop. */
2519 156564 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2520 13389 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2521 : {
2522 13373 : poly_uint64 unscaled_vf
2523 13373 : = exact_div (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
2524 : orig_loop_vinfo->suggested_unroll_factor);
2525 13373 : if (maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo), unscaled_vf))
2526 378 : return opt_result::failure_at (vect_location,
2527 : "Vectorization factor too high for"
2528 : " epilogue loop.\n");
2529 : }
2530 :
2531 : /* If the epilogue needs peeling for gaps but the main loop doesn't give
2532 : up on the epilogue. */
2533 156186 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2534 13011 : && LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2535 73 : && (LOOP_VINFO_PEELING_FOR_GAPS (orig_loop_vinfo)
2536 : != LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
2537 4 : return opt_result::failure_at (vect_location,
2538 : "Epilogue loop requires peeling for gaps "
2539 : "but main loop does not.\n");
2540 :
2541 : /* If an epilogue loop is required make sure we can create one. */
2542 156182 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2543 154886 : || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2544 56951 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2545 : {
2546 100761 : if (dump_enabled_p ())
2547 5563 : dump_printf_loc (MSG_NOTE, vect_location, "epilog loop required\n");
2548 100761 : if (!vect_can_advance_ivs_p (loop_vinfo)
2549 200994 : || !slpeel_can_duplicate_loop_p (loop,
2550 : LOOP_VINFO_MAIN_EXIT (loop_vinfo),
2551 100233 : LOOP_VINFO_MAIN_EXIT (loop_vinfo)))
2552 : {
2553 528 : ok = opt_result::failure_at (vect_location,
2554 : "not vectorized: can't create required "
2555 : "epilog loop\n");
2556 528 : goto again;
2557 : }
2558 : }
2559 :
2560 : /* Check the costings of the loop make vectorizing worthwhile. */
2561 155654 : res = vect_analyze_loop_costing (loop_vinfo, suggested_unroll_factor);
2562 155654 : if (res < 0 && !param_vect_allow_possibly_not_worthwhile_vectorizations)
2563 : {
2564 28780 : ok = opt_result::failure_at (vect_location,
2565 : "Loop costings may not be worthwhile.\n");
2566 28780 : goto again;
2567 : }
2568 126874 : if (!res)
2569 31755 : return opt_result::failure_at (vect_location,
2570 : "Loop costings not worthwhile.\n");
2571 :
2572 : /* During peeling, we need to check if number of loop iterations is
2573 : enough for both peeled prolog loop and vector loop. This check
2574 : can be merged along with threshold check of loop versioning, so
2575 : increase threshold for this case if necessary.
2576 :
2577 : If we are analyzing an epilogue we still want to check what its
2578 : versioning threshold would be. If we decide to vectorize the epilogues we
2579 : will want to use the lowest versioning threshold of all epilogues and main
2580 : loop. This will enable us to enter a vectorized epilogue even when
2581 : versioning the loop. We can't simply check whether the epilogue requires
2582 : versioning though since we may have skipped some versioning checks when
2583 : analyzing the epilogue. For instance, checks for alias versioning will be
2584 : skipped when dealing with epilogues as we assume we already checked them
2585 : for the main loop. So instead we always check the 'orig_loop_vinfo'. */
2586 95119 : if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo))
2587 : {
2588 9011 : poly_uint64 niters_th = 0;
2589 9011 : unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2590 :
2591 9011 : if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
2592 : {
2593 : /* Niters for peeled prolog loop. */
2594 9011 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
2595 : {
2596 115 : dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
2597 115 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
2598 115 : niters_th += TYPE_VECTOR_SUBPARTS (vectype) - 1;
2599 : }
2600 : else
2601 8896 : niters_th += LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
2602 : }
2603 :
2604 : /* Niters for at least one iteration of vectorized loop. */
2605 9011 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2606 9007 : niters_th += LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2607 : /* One additional iteration because of peeling for gap. */
2608 9011 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
2609 71 : niters_th += 1;
2610 :
2611 : /* Use the same condition as vect_transform_loop to decide when to use
2612 : the cost to determine a versioning threshold. */
2613 9011 : if (vect_apply_runtime_profitability_check_p (loop_vinfo)
2614 9011 : && ordered_p (th, niters_th))
2615 6687 : niters_th = ordered_max (poly_uint64 (th), niters_th);
2616 :
2617 9011 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = niters_th;
2618 : }
2619 :
2620 95119 : gcc_assert (known_eq (vectorization_factor,
2621 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
2622 :
2623 95119 : single_lane_slp_done_for_suggested_uf = force_single_lane;
2624 :
2625 : /* Ok to vectorize! */
2626 95119 : LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
2627 95119 : return opt_result::success ();
2628 :
2629 288536 : again:
2630 : /* Ensure that "ok" is false (with an opt_problem if dumping is enabled). */
2631 288536 : gcc_assert (!ok);
2632 :
2633 : /* Try again with single-lane SLP. */
2634 288536 : if (force_single_lane)
2635 142945 : return ok;
2636 :
2637 : /* If we are applying suggested unroll factor, we don't need to
2638 : re-try any more as we want to keep the SLP mode fixed. */
2639 145591 : if (applying_suggested_uf)
2640 10 : return ok;
2641 :
2642 : /* Likewise if the grouped loads or stores in the SLP cannot be handled
2643 : via interleaving or lane instructions. */
2644 : slp_instance instance;
2645 : slp_tree node;
2646 : unsigned i, j;
2647 392009 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
2648 : {
2649 247398 : if (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance)) != vect_internal_def)
2650 0 : continue;
2651 :
2652 247398 : stmt_vec_info vinfo;
2653 247398 : vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0];
2654 247398 : if (!vinfo || !STMT_VINFO_GROUPED_ACCESS (vinfo))
2655 244594 : continue;
2656 2804 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2657 2804 : unsigned int size = DR_GROUP_SIZE (vinfo);
2658 2804 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
2659 2804 : if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
2660 4932 : && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
2661 5602 : && ! vect_grouped_store_supported (vectype, size))
2662 670 : return opt_result::failure_at (vinfo->stmt,
2663 : "unsupported grouped store\n");
2664 250053 : FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node)
2665 : {
2666 2330 : vinfo = SLP_TREE_REPRESENTATIVE (node);
2667 2330 : if (STMT_VINFO_GROUPED_ACCESS (vinfo))
2668 : {
2669 2020 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2670 2020 : bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
2671 2020 : size = DR_GROUP_SIZE (vinfo);
2672 2020 : vectype = SLP_TREE_VECTYPE (node);
2673 2020 : if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
2674 2020 : && ! vect_grouped_load_supported (vectype, single_element_p,
2675 : size))
2676 300 : return opt_result::failure_at (vinfo->stmt,
2677 : "unsupported grouped load\n");
2678 : }
2679 : }
2680 : }
2681 :
2682 : /* Roll back state appropriately. Force single-lane SLP this time. */
2683 144611 : force_single_lane = true;
2684 144611 : if (dump_enabled_p ())
2685 3554 : dump_printf_loc (MSG_NOTE, vect_location,
2686 : "re-trying with single-lane SLP\n");
2687 :
2688 : /* Reset the vectorization factor. */
2689 144611 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
2690 : /* Free the SLP instances. */
2691 391032 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
2692 246421 : vect_free_slp_instance (instance);
2693 144611 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
2694 : /* Reset altered state on stmts. */
2695 692788 : for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
2696 : {
2697 403566 : basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
2698 403566 : for (gimple_stmt_iterator si = gsi_start_phis (bb);
2699 729795 : !gsi_end_p (si); gsi_next (&si))
2700 : {
2701 326229 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
2702 326229 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
2703 326229 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
2704 : {
2705 : /* vectorizable_reduction adjusts reduction stmt def-types,
2706 : restore them to that of the PHI. */
2707 26488 : STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info))
2708 26488 : = STMT_VINFO_DEF_TYPE (stmt_info);
2709 26488 : STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
2710 : (STMT_VINFO_REDUC_DEF (stmt_info)))
2711 26488 : = STMT_VINFO_DEF_TYPE (stmt_info);
2712 : }
2713 : }
2714 : }
2715 : /* Free optimized alias test DDRS. */
2716 144611 : LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
2717 144611 : LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
2718 144611 : LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).release ();
2719 : /* Reset target cost data. */
2720 144611 : delete loop_vinfo->vector_costs;
2721 144611 : loop_vinfo->vector_costs = nullptr;
2722 : /* Reset accumulated rgroup information. */
2723 144611 : LOOP_VINFO_MASKS (loop_vinfo).mask_set.empty ();
2724 144611 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
2725 144611 : release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo));
2726 : /* Reset assorted flags. */
2727 144611 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo) = false;
2728 144611 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = false;
2729 144611 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = 0;
2730 144611 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = 0;
2731 144611 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2732 144611 : = saved_can_use_partial_vectors_p;
2733 144611 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2734 144611 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2735 144611 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2736 144611 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
2737 :
2738 144611 : if (loop_vinfo->scan_map)
2739 122 : loop_vinfo->scan_map->empty ();
2740 :
2741 144611 : goto start_over;
2742 : }
2743 :
2744 : /* Return true if vectorizing a loop using NEW_LOOP_VINFO appears
2745 : to be better than vectorizing it using OLD_LOOP_VINFO. Assume that
2746 : OLD_LOOP_VINFO is better unless something specifically indicates
2747 : otherwise.
2748 :
2749 : Note that this deliberately isn't a partial order. */
2750 :
2751 : static bool
2752 32564 : vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo,
2753 : loop_vec_info old_loop_vinfo)
2754 : {
2755 32564 : struct loop *loop = LOOP_VINFO_LOOP (new_loop_vinfo);
2756 32564 : gcc_assert (LOOP_VINFO_LOOP (old_loop_vinfo) == loop);
2757 :
2758 32564 : poly_int64 new_vf = LOOP_VINFO_VECT_FACTOR (new_loop_vinfo);
2759 32564 : poly_int64 old_vf = LOOP_VINFO_VECT_FACTOR (old_loop_vinfo);
2760 :
2761 : /* Always prefer a VF of loop->simdlen over any other VF. */
2762 32564 : if (loop->simdlen)
2763 : {
2764 0 : bool new_simdlen_p = known_eq (new_vf, loop->simdlen);
2765 0 : bool old_simdlen_p = known_eq (old_vf, loop->simdlen);
2766 0 : if (new_simdlen_p != old_simdlen_p)
2767 : return new_simdlen_p;
2768 : }
2769 :
2770 32564 : const auto *old_costs = old_loop_vinfo->vector_costs;
2771 32564 : const auto *new_costs = new_loop_vinfo->vector_costs;
2772 32564 : if (loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo))
2773 1480 : return new_costs->better_epilogue_loop_than_p (old_costs, main_loop);
2774 :
2775 31084 : return new_costs->better_main_loop_than_p (old_costs);
2776 : }
2777 :
2778 : /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO. Return
2779 : true if we should. */
2780 :
2781 : static bool
2782 32564 : vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
2783 : loop_vec_info old_loop_vinfo)
2784 : {
2785 32564 : if (!vect_better_loop_vinfo_p (new_loop_vinfo, old_loop_vinfo))
2786 : return false;
2787 :
2788 1382 : if (dump_enabled_p ())
2789 18 : dump_printf_loc (MSG_NOTE, vect_location,
2790 : "***** Preferring vector mode %s to vector mode %s\n",
2791 18 : GET_MODE_NAME (new_loop_vinfo->vector_mode),
2792 18 : GET_MODE_NAME (old_loop_vinfo->vector_mode));
2793 : return true;
2794 : }
2795 :
2796 : /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
2797 : not NULL. When MASKED_P is not -1 override the default
2798 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
2799 : Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
2800 : mode useful to analyze.
2801 : Return the loop_vinfo on success and wrapped null on failure. */
2802 :
2803 : static opt_loop_vec_info
2804 588238 : vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
2805 : const vect_loop_form_info *loop_form_info,
2806 : loop_vec_info orig_loop_vinfo,
2807 : const vector_modes &vector_modes, unsigned &mode_i,
2808 : int masked_p,
2809 : machine_mode &autodetected_vector_mode,
2810 : bool &fatal)
2811 : {
2812 588238 : loop_vec_info loop_vinfo
2813 588238 : = vect_create_loop_vinfo (loop, shared, loop_form_info, orig_loop_vinfo);
2814 :
2815 588238 : machine_mode vector_mode = vector_modes[mode_i];
2816 588238 : loop_vinfo->vector_mode = vector_mode;
2817 588238 : unsigned int suggested_unroll_factor = 1;
2818 588238 : bool single_lane_slp_done_for_suggested_uf = false;
2819 :
2820 : /* Run the main analysis. */
2821 588238 : opt_result res = vect_analyze_loop_2 (loop_vinfo, masked_p, fatal,
2822 : &suggested_unroll_factor,
2823 : single_lane_slp_done_for_suggested_uf);
2824 588238 : if (dump_enabled_p ())
2825 21345 : dump_printf_loc (MSG_NOTE, vect_location,
2826 : "***** Analysis %s with vector mode %s\n",
2827 21345 : res ? "succeeded" : "failed",
2828 21345 : GET_MODE_NAME (loop_vinfo->vector_mode));
2829 :
2830 588238 : auto user_unroll = LOOP_VINFO_LOOP (loop_vinfo)->unroll;
2831 588238 : if (res && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2832 : /* Check to see if the user wants to unroll or if the target wants to. */
2833 674623 : && (suggested_unroll_factor > 1 || user_unroll > 1))
2834 : {
2835 485 : if (suggested_unroll_factor == 1)
2836 : {
2837 66 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
2838 66 : suggested_unroll_factor = user_unroll / assumed_vf;
2839 66 : if (suggested_unroll_factor > 1)
2840 : {
2841 40 : if (dump_enabled_p ())
2842 20 : dump_printf_loc (MSG_NOTE, vect_location,
2843 : "setting unroll factor to %d based on user requested "
2844 : "unroll factor %d and suggested vectorization "
2845 : "factor: %d\n",
2846 : suggested_unroll_factor, user_unroll, assumed_vf);
2847 : }
2848 : }
2849 :
2850 485 : if (suggested_unroll_factor > 1)
2851 : {
2852 459 : if (dump_enabled_p ())
2853 62 : dump_printf_loc (MSG_NOTE, vect_location,
2854 : "***** Re-trying analysis for unrolling"
2855 : " with unroll factor %d and %s slp.\n",
2856 : suggested_unroll_factor,
2857 : single_lane_slp_done_for_suggested_uf
2858 : ? "single-lane" : "");
2859 459 : loop_vec_info unroll_vinfo
2860 459 : = vect_create_loop_vinfo (loop, shared, loop_form_info, NULL);
2861 459 : unroll_vinfo->vector_mode = vector_mode;
2862 459 : unroll_vinfo->suggested_unroll_factor = suggested_unroll_factor;
2863 459 : opt_result new_res
2864 459 : = vect_analyze_loop_2 (unroll_vinfo, masked_p, fatal, NULL,
2865 : single_lane_slp_done_for_suggested_uf);
2866 459 : if (new_res)
2867 : {
2868 400 : delete loop_vinfo;
2869 : loop_vinfo = unroll_vinfo;
2870 : }
2871 : else
2872 59 : delete unroll_vinfo;
2873 : }
2874 :
2875 : /* Record that we have honored a user unroll factor. */
2876 485 : LOOP_VINFO_USER_UNROLL (loop_vinfo) = user_unroll > 1;
2877 : }
2878 :
2879 : /* Remember the autodetected vector mode. */
2880 588238 : if (vector_mode == VOIDmode)
2881 272557 : autodetected_vector_mode = loop_vinfo->vector_mode;
2882 :
2883 : /* Advance mode_i, first skipping modes that would result in the
2884 : same analysis result. */
2885 2594080 : while (mode_i + 1 < vector_modes.length ()
2886 1784288 : && vect_chooses_same_modes_p (loop_vinfo,
2887 781367 : vector_modes[mode_i + 1]))
2888 : {
2889 414683 : if (dump_enabled_p ())
2890 17115 : dump_printf_loc (MSG_NOTE, vect_location,
2891 : "***** The result for vector mode %s would"
2892 : " be the same\n",
2893 17115 : GET_MODE_NAME (vector_modes[mode_i + 1]));
2894 414683 : mode_i += 1;
2895 : }
2896 588238 : if (mode_i + 1 < vector_modes.length ()
2897 954922 : && vect_chooses_same_modes_p (autodetected_vector_mode,
2898 366684 : vector_modes[mode_i + 1]))
2899 : {
2900 426 : if (dump_enabled_p ())
2901 11 : dump_printf_loc (MSG_NOTE, vect_location,
2902 : "***** Skipping vector mode %s, which would"
2903 : " repeat the analysis for %s\n",
2904 11 : GET_MODE_NAME (vector_modes[mode_i + 1]),
2905 11 : GET_MODE_NAME (autodetected_vector_mode));
2906 426 : mode_i += 1;
2907 : }
2908 588238 : mode_i++;
2909 :
2910 588238 : if (!res)
2911 : {
2912 493519 : delete loop_vinfo;
2913 493519 : if (fatal)
2914 104384 : gcc_checking_assert (orig_loop_vinfo == NULL);
2915 493519 : return opt_loop_vec_info::propagate_failure (res);
2916 : }
2917 :
2918 94719 : return opt_loop_vec_info::success (loop_vinfo);
2919 : }
2920 :
2921 : /* Function vect_analyze_loop.
2922 :
2923 : Apply a set of analyses on LOOP, and create a loop_vec_info struct
2924 : for it. The different analyses will record information in the
2925 : loop_vec_info struct. */
2926 : opt_loop_vec_info
2927 473847 : vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
2928 : vec_info_shared *shared)
2929 : {
2930 473847 : DUMP_VECT_SCOPE ("analyze_loop_nest");
2931 :
2932 473847 : if (loop_outer (loop)
2933 473847 : && loop_vec_info_for_loop (loop_outer (loop))
2934 474429 : && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
2935 582 : return opt_loop_vec_info::failure_at (vect_location,
2936 : "outer-loop already vectorized.\n");
2937 :
2938 473265 : if (!find_loop_nest (loop, &shared->loop_nest))
2939 20924 : return opt_loop_vec_info::failure_at
2940 20924 : (vect_location,
2941 : "not vectorized: loop nest containing two or more consecutive inner"
2942 : " loops cannot be vectorized\n");
2943 :
2944 : /* Analyze the loop form. */
2945 452341 : vect_loop_form_info loop_form_info;
2946 452341 : opt_result res = vect_analyze_loop_form (loop, loop_vectorized_call,
2947 : &loop_form_info);
2948 452341 : if (!res)
2949 : {
2950 179784 : if (dump_enabled_p ())
2951 1527 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2952 : "bad loop form.\n");
2953 179784 : return opt_loop_vec_info::propagate_failure (res);
2954 : }
2955 272557 : if (!integer_onep (loop_form_info.assumptions))
2956 : {
2957 : /* We consider to vectorize this loop by versioning it under
2958 : some assumptions. In order to do this, we need to clear
2959 : existing information computed by scev and niter analyzer. */
2960 8553 : scev_reset_htab ();
2961 8553 : free_numbers_of_iterations_estimates (loop);
2962 : /* Also set flag for this loop so that following scev and niter
2963 : analysis are done under the assumptions. */
2964 8553 : loop_constraint_set (loop, LOOP_C_FINITE);
2965 : }
2966 : else
2967 : /* Clear the existing niter information to make sure the nonwrapping flag
2968 : will be calculated and set propriately. */
2969 264004 : free_numbers_of_iterations_estimates (loop);
2970 :
2971 272557 : auto_vector_modes vector_modes;
2972 : /* Autodetect first vector size we try. */
2973 272557 : vector_modes.safe_push (VOIDmode);
2974 272557 : unsigned int autovec_flags
2975 545114 : = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
2976 272557 : loop->simdlen != 0);
2977 272557 : bool pick_lowest_cost_p = ((autovec_flags & VECT_COMPARE_COSTS)
2978 272557 : && !unlimited_cost_model (loop));
2979 272557 : machine_mode autodetected_vector_mode = VOIDmode;
2980 272557 : opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
2981 272557 : unsigned int mode_i = 0;
2982 272557 : unsigned HOST_WIDE_INT simdlen = loop->simdlen;
2983 :
2984 : /* Keep track of the VF for each mode. Initialize all to 0 which indicates
2985 : a mode has not been analyzed. */
2986 272557 : auto_vec<poly_uint64, 8> cached_vf_per_mode;
2987 2737530 : for (unsigned i = 0; i < vector_modes.length (); ++i)
2988 1096208 : cached_vf_per_mode.safe_push (0);
2989 :
2990 : /* First determine the main loop vectorization mode, either the first
2991 : one that works, starting with auto-detecting the vector mode and then
2992 : following the targets order of preference, or the one with the
2993 : lowest cost if pick_lowest_cost_p. */
2994 876293 : while (1)
2995 : {
2996 574425 : bool fatal;
2997 574425 : unsigned int last_mode_i = mode_i;
2998 : /* Set cached VF to -1 prior to analysis, which indicates a mode has
2999 : failed. */
3000 574425 : cached_vf_per_mode[last_mode_i] = -1;
3001 574425 : opt_loop_vec_info loop_vinfo
3002 574425 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3003 : NULL, vector_modes, mode_i, -1,
3004 : autodetected_vector_mode, fatal);
3005 574425 : if (fatal)
3006 : break;
3007 :
3008 470041 : if (loop_vinfo)
3009 : {
3010 : /* Analysis has been successful so update the VF value. The
3011 : VF should always be a multiple of unroll_factor and we want to
3012 : capture the original VF here. */
3013 86385 : cached_vf_per_mode[last_mode_i]
3014 86385 : = exact_div (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
3015 86385 : loop_vinfo->suggested_unroll_factor);
3016 : /* Once we hit the desired simdlen for the first time,
3017 : discard any previous attempts. */
3018 86385 : if (simdlen
3019 86385 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), simdlen))
3020 : {
3021 47 : delete first_loop_vinfo;
3022 86385 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3023 86385 : simdlen = 0;
3024 : }
3025 86338 : else if (pick_lowest_cost_p
3026 72334 : && first_loop_vinfo
3027 117422 : && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
3028 : {
3029 : /* Pick loop_vinfo over first_loop_vinfo. */
3030 1212 : delete first_loop_vinfo;
3031 1212 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3032 : }
3033 86385 : if (first_loop_vinfo == NULL)
3034 : first_loop_vinfo = loop_vinfo;
3035 : else
3036 : {
3037 29874 : delete loop_vinfo;
3038 29874 : loop_vinfo = opt_loop_vec_info::success (NULL);
3039 : }
3040 :
3041 : /* Commit to first_loop_vinfo if we have no reason to try
3042 : alternatives. */
3043 86385 : if (!simdlen && !pick_lowest_cost_p)
3044 : break;
3045 : }
3046 455999 : if (mode_i == vector_modes.length ()
3047 455999 : || autodetected_vector_mode == VOIDmode)
3048 : break;
3049 :
3050 : /* Try the next biggest vector size. */
3051 301868 : if (dump_enabled_p ())
3052 4811 : dump_printf_loc (MSG_NOTE, vect_location,
3053 : "***** Re-trying analysis with vector mode %s\n",
3054 4811 : GET_MODE_NAME (vector_modes[mode_i]));
3055 301868 : }
3056 272557 : if (!first_loop_vinfo)
3057 217263 : return opt_loop_vec_info::propagate_failure (res);
3058 :
3059 55294 : if (dump_enabled_p ())
3060 9591 : dump_printf_loc (MSG_NOTE, vect_location,
3061 : "***** Choosing vector mode %s\n",
3062 9591 : GET_MODE_NAME (first_loop_vinfo->vector_mode));
3063 :
3064 : /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
3065 : enabled, SIMDUID is not set, it is the innermost loop and we have
3066 : either already found the loop's SIMDLEN or there was no SIMDLEN to
3067 : begin with.
3068 : TODO: Enable epilogue vectorization for loops with SIMDUID set. */
3069 55294 : bool vect_epilogues = (!simdlen
3070 55292 : && loop->inner == NULL
3071 54697 : && param_vect_epilogues_nomask
3072 53555 : && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
3073 : /* No code motion support for multiple epilogues so for now
3074 : not supported when multiple exits. */
3075 26179 : && !LOOP_VINFO_EARLY_BREAKS (first_loop_vinfo)
3076 25679 : && !loop->simduid
3077 79583 : && loop_cost_model (loop) > VECT_COST_MODEL_VERY_CHEAP);
3078 55294 : if (!vect_epilogues)
3079 42338 : return first_loop_vinfo;
3080 :
3081 : /* Now analyze first_loop_vinfo for epilogue vectorization. */
3082 :
3083 : /* For epilogues start the analysis from the first mode. The motivation
3084 : behind starting from the beginning comes from cases where the VECTOR_MODES
3085 : array may contain length-agnostic and length-specific modes. Their
3086 : ordering is not guaranteed, so we could end up picking a mode for the main
3087 : loop that is after the epilogue's optimal mode. */
3088 12956 : int masked_p = -1;
3089 12956 : if (!unlimited_cost_model (loop)
3090 12956 : && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3091 : != VOIDmode))
3092 : {
3093 5 : vector_modes[0]
3094 5 : = first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3095 5 : cached_vf_per_mode[0] = 0;
3096 : }
3097 : else
3098 12951 : vector_modes[0] = autodetected_vector_mode;
3099 12956 : mode_i = 0;
3100 :
3101 12993 : bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
3102 12956 : || masked_p == 1);
3103 : if (supports_partial_vectors
3104 37 : && !partial_vectors_supported_p ()
3105 37 : && !LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (first_loop_vinfo))
3106 : supports_partial_vectors = false;
3107 12956 : poly_uint64 first_vinfo_vf = LOOP_VINFO_VECT_FACTOR (first_loop_vinfo);
3108 :
3109 12956 : loop_vec_info orig_loop_vinfo = first_loop_vinfo;
3110 13120 : do
3111 : {
3112 : /* Let the user override what the target suggests. */
3113 13038 : if (OPTION_SET_P (param_vect_partial_vector_usage))
3114 45 : masked_p = -1;
3115 :
3116 50457 : while (1)
3117 : {
3118 : /* If the target does not support partial vectors we can shorten the
3119 : number of modes to analyze for the epilogue as we know we can't
3120 : pick a mode that would lead to a VF at least as big as the
3121 : FIRST_VINFO_VF. */
3122 67170 : if (!supports_partial_vectors
3123 50457 : && maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
3124 : {
3125 23885 : mode_i++;
3126 47770 : if (mode_i == vector_modes.length ())
3127 : break;
3128 29472 : continue;
3129 : }
3130 : /* We would need an exhaustive search to find all modes we
3131 : skipped but that would lead to the same result as the
3132 : analysis it was skipped for and where we'd could check
3133 : cached_vf_per_mode against.
3134 : Check for the autodetected mode, which is the common
3135 : situation on x86 which does not perform cost comparison. */
3136 39331 : if (!supports_partial_vectors
3137 26524 : && maybe_ge (cached_vf_per_mode[0], first_vinfo_vf)
3138 52313 : && vect_chooses_same_modes_p (autodetected_vector_mode,
3139 25741 : vector_modes[mode_i]))
3140 : {
3141 12759 : mode_i++;
3142 25518 : if (mode_i == vector_modes.length ())
3143 : break;
3144 12759 : continue;
3145 : }
3146 :
3147 13813 : if (dump_enabled_p ())
3148 3255 : dump_printf_loc (MSG_NOTE, vect_location,
3149 : "***** Re-trying epilogue analysis with vector "
3150 3255 : "mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
3151 :
3152 13813 : bool fatal;
3153 13813 : opt_loop_vec_info loop_vinfo
3154 13813 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3155 : orig_loop_vinfo,
3156 : vector_modes, mode_i, masked_p,
3157 : autodetected_vector_mode, fatal);
3158 13813 : if (fatal)
3159 : break;
3160 :
3161 13813 : if (loop_vinfo)
3162 : {
3163 8334 : if (pick_lowest_cost_p
3164 5395 : && orig_loop_vinfo->epilogue_vinfo
3165 9814 : && vect_joust_loop_vinfos (loop_vinfo,
3166 1480 : orig_loop_vinfo->epilogue_vinfo))
3167 : {
3168 170 : gcc_assert (vect_epilogues);
3169 170 : delete orig_loop_vinfo->epilogue_vinfo;
3170 170 : orig_loop_vinfo->epilogue_vinfo = nullptr;
3171 : }
3172 8334 : if (!orig_loop_vinfo->epilogue_vinfo)
3173 7024 : orig_loop_vinfo->epilogue_vinfo = loop_vinfo;
3174 : else
3175 : {
3176 1310 : delete loop_vinfo;
3177 1310 : loop_vinfo = opt_loop_vec_info::success (NULL);
3178 : }
3179 :
3180 : /* For now only allow one epilogue loop, but allow
3181 : pick_lowest_cost_p to replace it, so commit to the
3182 : first epilogue if we have no reason to try alternatives. */
3183 8334 : if (!pick_lowest_cost_p)
3184 : break;
3185 : }
3186 :
3187 : /* Revert back to the default from the suggested preferred
3188 : epilogue vectorization mode. */
3189 10874 : masked_p = -1;
3190 21748 : if (mode_i == vector_modes.length ())
3191 : break;
3192 : }
3193 :
3194 13038 : orig_loop_vinfo = orig_loop_vinfo->epilogue_vinfo;
3195 13038 : if (!orig_loop_vinfo)
3196 : break;
3197 :
3198 : /* When we selected a first vectorized epilogue, see if the target
3199 : suggests to have another one. */
3200 6854 : masked_p = -1;
3201 6854 : if (!unlimited_cost_model (loop)
3202 3921 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
3203 10768 : && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3204 : != VOIDmode))
3205 : {
3206 164 : vector_modes[0]
3207 82 : = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3208 82 : cached_vf_per_mode[0] = 0;
3209 82 : mode_i = 0;
3210 : }
3211 : else
3212 : break;
3213 82 : }
3214 : while (1);
3215 :
3216 12956 : if (first_loop_vinfo->epilogue_vinfo)
3217 : {
3218 6779 : poly_uint64 lowest_th
3219 6779 : = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
3220 6779 : loop_vec_info epilog_vinfo = first_loop_vinfo->epilogue_vinfo;
3221 6854 : do
3222 : {
3223 6854 : poly_uint64 th = LOOP_VINFO_VERSIONING_THRESHOLD (epilog_vinfo);
3224 6854 : gcc_assert (!LOOP_REQUIRES_VERSIONING (epilog_vinfo)
3225 : || maybe_ne (lowest_th, 0U));
3226 : /* Keep track of the known smallest versioning threshold. */
3227 6854 : if (ordered_p (lowest_th, th))
3228 6854 : lowest_th = ordered_min (lowest_th, th);
3229 6854 : epilog_vinfo = epilog_vinfo->epilogue_vinfo;
3230 : }
3231 6854 : while (epilog_vinfo);
3232 6779 : LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
3233 6779 : if (dump_enabled_p ())
3234 1443 : dump_printf_loc (MSG_NOTE, vect_location,
3235 : "***** Choosing epilogue vector mode %s\n",
3236 1443 : GET_MODE_NAME
3237 : (first_loop_vinfo->epilogue_vinfo->vector_mode));
3238 : }
3239 :
3240 12956 : return first_loop_vinfo;
3241 724898 : }
3242 :
3243 : /* Return true if there is an in-order reduction function for CODE, storing
3244 : it in *REDUC_FN if so. */
3245 :
3246 : static bool
3247 5100 : fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
3248 : {
3249 : /* We support MINUS_EXPR by negating the operand. This also preserves an
3250 : initial -0.0 since -0.0 - 0.0 (neutral op for MINUS_EXPR) == -0.0 +
3251 : (-0.0) = -0.0. */
3252 5100 : if (code == PLUS_EXPR || code == MINUS_EXPR)
3253 : {
3254 4428 : *reduc_fn = IFN_FOLD_LEFT_PLUS;
3255 0 : return true;
3256 : }
3257 : return false;
3258 : }
3259 :
3260 : /* Function reduction_fn_for_scalar_code
3261 :
3262 : Input:
3263 : CODE - tree_code of a reduction operations.
3264 :
3265 : Output:
3266 : REDUC_FN - the corresponding internal function to be used to reduce the
3267 : vector of partial results into a single scalar result, or IFN_LAST
3268 : if the operation is a supported reduction operation, but does not have
3269 : such an internal function.
3270 :
3271 : Return FALSE if CODE currently cannot be vectorized as reduction. */
3272 :
3273 : bool
3274 2245268 : reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
3275 : {
3276 2245268 : if (code.is_tree_code ())
3277 2245210 : switch (tree_code (code))
3278 : {
3279 13716 : case MAX_EXPR:
3280 13716 : *reduc_fn = IFN_REDUC_MAX;
3281 13716 : return true;
3282 :
3283 61109 : case MIN_EXPR:
3284 61109 : *reduc_fn = IFN_REDUC_MIN;
3285 61109 : return true;
3286 :
3287 1232053 : case PLUS_EXPR:
3288 1232053 : *reduc_fn = IFN_REDUC_PLUS;
3289 1232053 : return true;
3290 :
3291 235974 : case BIT_AND_EXPR:
3292 235974 : *reduc_fn = IFN_REDUC_AND;
3293 235974 : return true;
3294 :
3295 259107 : case BIT_IOR_EXPR:
3296 259107 : *reduc_fn = IFN_REDUC_IOR;
3297 259107 : return true;
3298 :
3299 44026 : case BIT_XOR_EXPR:
3300 44026 : *reduc_fn = IFN_REDUC_XOR;
3301 44026 : return true;
3302 :
3303 399225 : case MULT_EXPR:
3304 399225 : case MINUS_EXPR:
3305 399225 : *reduc_fn = IFN_LAST;
3306 399225 : return true;
3307 :
3308 : default:
3309 : return false;
3310 : }
3311 : else
3312 58 : switch (combined_fn (code))
3313 : {
3314 34 : CASE_CFN_FMAX:
3315 34 : *reduc_fn = IFN_REDUC_FMAX;
3316 34 : return true;
3317 :
3318 24 : CASE_CFN_FMIN:
3319 24 : *reduc_fn = IFN_REDUC_FMIN;
3320 24 : return true;
3321 :
3322 : default:
3323 : return false;
3324 : }
3325 : }
3326 :
3327 : /* Set *SBOOL_FN to the corresponding function working on vector masks
3328 : for REDUC_FN. Return true if that exists, false otherwise. */
3329 :
3330 : static bool
3331 0 : sbool_reduction_fn_for_fn (internal_fn reduc_fn, internal_fn *sbool_fn)
3332 : {
3333 0 : switch (reduc_fn)
3334 : {
3335 0 : case IFN_REDUC_AND:
3336 0 : *sbool_fn = IFN_REDUC_SBOOL_AND;
3337 0 : return true;
3338 0 : case IFN_REDUC_IOR:
3339 0 : *sbool_fn = IFN_REDUC_SBOOL_IOR;
3340 0 : return true;
3341 0 : case IFN_REDUC_XOR:
3342 0 : *sbool_fn = IFN_REDUC_SBOOL_XOR;
3343 0 : return true;
3344 : default:
3345 : return false;
3346 : }
3347 : }
3348 :
3349 : /* If there is a neutral value X such that a reduction would not be affected
3350 : by the introduction of additional X elements, return that X, otherwise
3351 : return null. CODE is the code of the reduction and SCALAR_TYPE is type
3352 : of the scalar elements. If the reduction has just a single initial value
3353 : then INITIAL_VALUE is that value, otherwise it is null.
3354 : If AS_INITIAL is TRUE the value is supposed to be used as initial value.
3355 : In that case no signed zero is returned. */
3356 :
3357 : tree
3358 77968 : neutral_op_for_reduction (tree scalar_type, code_helper code,
3359 : tree initial_value, bool as_initial)
3360 : {
3361 77968 : if (code.is_tree_code ())
3362 77910 : switch (tree_code (code))
3363 : {
3364 13946 : case DOT_PROD_EXPR:
3365 13946 : case SAD_EXPR:
3366 13946 : case MINUS_EXPR:
3367 13946 : case BIT_IOR_EXPR:
3368 13946 : case BIT_XOR_EXPR:
3369 13946 : return build_zero_cst (scalar_type);
3370 57646 : case WIDEN_SUM_EXPR:
3371 57646 : case PLUS_EXPR:
3372 57646 : if (!as_initial && HONOR_SIGNED_ZEROS (scalar_type))
3373 92 : return build_real (scalar_type, dconstm0);
3374 : else
3375 57554 : return build_zero_cst (scalar_type);
3376 :
3377 2259 : case MULT_EXPR:
3378 2259 : return build_one_cst (scalar_type);
3379 :
3380 1578 : case BIT_AND_EXPR:
3381 1578 : return build_all_ones_cst (scalar_type);
3382 :
3383 : case MAX_EXPR:
3384 : case MIN_EXPR:
3385 : return initial_value;
3386 :
3387 428 : default:
3388 428 : return NULL_TREE;
3389 : }
3390 : else
3391 58 : switch (combined_fn (code))
3392 : {
3393 : CASE_CFN_FMIN:
3394 : CASE_CFN_FMAX:
3395 : return initial_value;
3396 :
3397 0 : default:
3398 0 : return NULL_TREE;
3399 : }
3400 : }
3401 :
3402 : /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
3403 : STMT is printed with a message MSG. */
3404 :
3405 : static void
3406 578 : report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
3407 : {
3408 578 : dump_printf_loc (msg_type, vect_location, "%s%G", msg, stmt);
3409 578 : }
3410 :
3411 : /* Return true if we need an in-order reduction for operation CODE
3412 : on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
3413 : overflow must wrap. */
3414 :
3415 : bool
3416 359689 : needs_fold_left_reduction_p (tree type, code_helper code)
3417 : {
3418 : /* CHECKME: check for !flag_finite_math_only too? */
3419 359689 : if (SCALAR_FLOAT_TYPE_P (type))
3420 : {
3421 100115 : if (code.is_tree_code ())
3422 100061 : switch (tree_code (code))
3423 : {
3424 : case MIN_EXPR:
3425 : case MAX_EXPR:
3426 : return false;
3427 :
3428 99485 : default:
3429 99485 : return !flag_associative_math;
3430 : }
3431 : else
3432 54 : switch (combined_fn (code))
3433 : {
3434 : CASE_CFN_FMIN:
3435 : CASE_CFN_FMAX:
3436 : return false;
3437 :
3438 2 : default:
3439 2 : return !flag_associative_math;
3440 : }
3441 : }
3442 :
3443 259574 : if (INTEGRAL_TYPE_P (type))
3444 259464 : return (!code.is_tree_code ()
3445 259464 : || !operation_no_trapping_overflow (type, tree_code (code)));
3446 :
3447 110 : if (SAT_FIXED_POINT_TYPE_P (type))
3448 : return true;
3449 :
3450 : return false;
3451 : }
3452 :
3453 : /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
3454 : has a handled computation expression. Store the main reduction
3455 : operation in *CODE. */
3456 :
3457 : static bool
3458 102880 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3459 : tree loop_arg, code_helper *code,
3460 : vec<std::pair<ssa_op_iter, use_operand_p> > &path,
3461 : bool inner_loop_of_double_reduc)
3462 : {
3463 102880 : auto_bitmap visited;
3464 102880 : tree lookfor = PHI_RESULT (phi);
3465 102880 : ssa_op_iter curri;
3466 102880 : use_operand_p curr = op_iter_init_phiuse (&curri, phi, SSA_OP_USE);
3467 213937 : while (USE_FROM_PTR (curr) != loop_arg)
3468 8177 : curr = op_iter_next_use (&curri);
3469 102880 : curri.i = curri.numops;
3470 951890 : do
3471 : {
3472 951890 : path.safe_push (std::make_pair (curri, curr));
3473 951890 : tree use = USE_FROM_PTR (curr);
3474 951890 : if (use == lookfor)
3475 : break;
3476 849439 : gimple *def = SSA_NAME_DEF_STMT (use);
3477 849439 : if (gimple_nop_p (def)
3478 849439 : || ! flow_bb_inside_loop_p (loop, gimple_bb (def)))
3479 : {
3480 239803 : pop:
3481 716958 : do
3482 : {
3483 716958 : std::pair<ssa_op_iter, use_operand_p> x = path.pop ();
3484 716958 : curri = x.first;
3485 716958 : curr = x.second;
3486 784793 : do
3487 784793 : curr = op_iter_next_use (&curri);
3488 : /* Skip already visited or non-SSA operands (from iterating
3489 : over PHI args). */
3490 : while (curr != NULL_USE_OPERAND_P
3491 1092002 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3492 271197 : || ! bitmap_set_bit (visited,
3493 271197 : SSA_NAME_VERSION
3494 : (USE_FROM_PTR (curr)))));
3495 : }
3496 1433916 : while (curr == NULL_USE_OPERAND_P && ! path.is_empty ());
3497 239803 : if (curr == NULL_USE_OPERAND_P)
3498 : break;
3499 : }
3500 : else
3501 : {
3502 714075 : if (gimple_code (def) == GIMPLE_PHI)
3503 72944 : curr = op_iter_init_phiuse (&curri, as_a <gphi *>(def), SSA_OP_USE);
3504 : else
3505 641131 : curr = op_iter_init_use (&curri, def, SSA_OP_USE);
3506 : while (curr != NULL_USE_OPERAND_P
3507 853276 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3508 743488 : || ! bitmap_set_bit (visited,
3509 743488 : SSA_NAME_VERSION
3510 : (USE_FROM_PTR (curr)))))
3511 139201 : curr = op_iter_next_use (&curri);
3512 714075 : if (curr == NULL_USE_OPERAND_P)
3513 104439 : goto pop;
3514 : }
3515 : }
3516 : while (1);
3517 102880 : if (dump_file && (dump_flags & TDF_DETAILS))
3518 : {
3519 4148 : dump_printf_loc (MSG_NOTE, loc, "reduction path: ");
3520 4148 : unsigned i;
3521 4148 : std::pair<ssa_op_iter, use_operand_p> *x;
3522 14092 : FOR_EACH_VEC_ELT (path, i, x)
3523 9944 : dump_printf (MSG_NOTE, "%T ", USE_FROM_PTR (x->second));
3524 4148 : dump_printf (MSG_NOTE, "\n");
3525 : }
3526 :
3527 : /* Check whether the reduction path detected is valid. */
3528 102880 : bool fail = path.length () == 0;
3529 102880 : bool neg = false;
3530 102880 : int sign = -1;
3531 102880 : *code = ERROR_MARK;
3532 224681 : for (unsigned i = 1; i < path.length (); ++i)
3533 : {
3534 125339 : gimple *use_stmt = USE_STMT (path[i].second);
3535 125339 : gimple_match_op op;
3536 125339 : if (!gimple_extract_op (use_stmt, &op))
3537 : {
3538 : fail = true;
3539 3538 : break;
3540 : }
3541 124423 : unsigned int opi = op.num_ops;
3542 124423 : if (gassign *assign = dyn_cast<gassign *> (use_stmt))
3543 : {
3544 : /* The following make sure we can compute the operand index
3545 : easily plus it mostly disallows chaining via COND_EXPR condition
3546 : operands. */
3547 192245 : for (opi = 0; opi < op.num_ops; ++opi)
3548 191215 : if (gimple_assign_rhs1_ptr (assign) + opi == path[i].second->use)
3549 : break;
3550 : }
3551 6237 : else if (gcall *call = dyn_cast<gcall *> (use_stmt))
3552 : {
3553 12483 : for (opi = 0; opi < op.num_ops; ++opi)
3554 12483 : if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
3555 : break;
3556 : }
3557 124423 : if (opi == op.num_ops)
3558 : {
3559 : fail = true;
3560 : break;
3561 : }
3562 123393 : op.code = canonicalize_code (op.code, op.type);
3563 123393 : if (op.code == MINUS_EXPR)
3564 : {
3565 5691 : op.code = PLUS_EXPR;
3566 : /* Track whether we negate the reduction value each iteration. */
3567 5691 : if (op.ops[1] == op.ops[opi])
3568 34 : neg = ! neg;
3569 : }
3570 117702 : else if (op.code == IFN_COND_SUB)
3571 : {
3572 9 : op.code = IFN_COND_ADD;
3573 : /* Track whether we negate the reduction value each iteration. */
3574 9 : if (op.ops[2] == op.ops[opi])
3575 0 : neg = ! neg;
3576 : }
3577 : /* For an FMA the reduction code is the PLUS if the addition chain
3578 : is the reduction. */
3579 117693 : else if (op.code == IFN_FMA && opi == 2)
3580 33 : op.code = PLUS_EXPR;
3581 123393 : if (CONVERT_EXPR_CODE_P (op.code)
3582 123393 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
3583 : ;
3584 117787 : else if (*code == ERROR_MARK)
3585 : {
3586 100615 : *code = op.code;
3587 100615 : sign = TYPE_SIGN (op.type);
3588 : }
3589 17172 : else if (op.code != *code)
3590 : {
3591 : fail = true;
3592 : break;
3593 : }
3594 15832 : else if ((op.code == MIN_EXPR
3595 15676 : || op.code == MAX_EXPR)
3596 15847 : && sign != TYPE_SIGN (op.type))
3597 : {
3598 : fail = true;
3599 : break;
3600 : }
3601 : /* Check there's only a single stmt the op is used on. For the
3602 : not value-changing tail and the last stmt allow out-of-loop uses,
3603 : but not when this is the inner loop of a double reduction.
3604 : ??? We could relax this and handle arbitrary live stmts by
3605 : forcing a scalar epilogue for example. */
3606 122050 : imm_use_iterator imm_iter;
3607 122050 : use_operand_p use_p;
3608 122050 : gimple *op_use_stmt;
3609 122050 : unsigned cnt = 0;
3610 128252 : bool cond_fn_p = op.code.is_internal_fn ()
3611 6202 : && (conditional_internal_fn_code (internal_fn (op.code))
3612 122050 : != ERROR_MARK);
3613 :
3614 293433 : FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
3615 : {
3616 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we should
3617 : have op1 twice (once as definition, once as else) in the same
3618 : operation. Enforce this. */
3619 171383 : if (cond_fn_p && op_use_stmt == use_stmt)
3620 : {
3621 6120 : gcall *call = as_a<gcall *> (use_stmt);
3622 6120 : unsigned else_pos
3623 6120 : = internal_fn_else_index (internal_fn (op.code));
3624 6120 : if (gimple_call_arg (call, else_pos) != op.ops[opi])
3625 : {
3626 : fail = true;
3627 : break;
3628 : }
3629 30600 : for (unsigned int j = 0; j < gimple_call_num_args (call); ++j)
3630 : {
3631 24480 : if (j == else_pos)
3632 6120 : continue;
3633 18360 : if (gimple_call_arg (call, j) == op.ops[opi])
3634 6120 : cnt++;
3635 : }
3636 : }
3637 165263 : else if (!is_gimple_debug (op_use_stmt)
3638 165263 : && ((*code != ERROR_MARK || inner_loop_of_double_reduc)
3639 2826 : || flow_bb_inside_loop_p (loop,
3640 2826 : gimple_bb (op_use_stmt))))
3641 232505 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
3642 116257 : cnt++;
3643 122050 : }
3644 :
3645 122050 : if (cnt != 1)
3646 : {
3647 : fail = true;
3648 : break;
3649 : }
3650 : }
3651 106858 : return ! fail && ! neg && *code != ERROR_MARK;
3652 102880 : }
3653 :
3654 : bool
3655 21 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3656 : tree loop_arg, enum tree_code code)
3657 : {
3658 21 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3659 21 : code_helper code_;
3660 21 : return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path, false)
3661 21 : && code_ == code);
3662 21 : }
3663 :
3664 :
3665 :
3666 : /* Function vect_is_simple_reduction
3667 :
3668 : (1) Detect a cross-iteration def-use cycle that represents a simple
3669 : reduction computation. We look for the following pattern:
3670 :
3671 : loop_header:
3672 : a1 = phi < a0, a2 >
3673 : a3 = ...
3674 : a2 = operation (a3, a1)
3675 :
3676 : or
3677 :
3678 : a3 = ...
3679 : loop_header:
3680 : a1 = phi < a0, a2 >
3681 : a2 = operation (a3, a1)
3682 :
3683 : such that:
3684 : 1. operation is commutative and associative and it is safe to
3685 : change the order of the computation
3686 : 2. no uses for a2 in the loop (a2 is used out of the loop)
3687 : 3. no uses of a1 in the loop besides the reduction operation
3688 : 4. no uses of a1 outside the loop.
3689 :
3690 : Conditions 1,4 are tested here.
3691 : Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
3692 :
3693 : (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
3694 : nested cycles.
3695 :
3696 : (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
3697 : reductions:
3698 :
3699 : a1 = phi < a0, a2 >
3700 : inner loop (def of a3)
3701 : a2 = phi < a3 >
3702 :
3703 : (4) Detect condition expressions, ie:
3704 : for (int i = 0; i < N; i++)
3705 : if (a[i] < val)
3706 : ret_val = a[i];
3707 :
3708 : */
3709 :
3710 : static stmt_vec_info
3711 166835 : vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
3712 : gphi **double_reduc)
3713 : {
3714 166835 : gphi *phi = as_a <gphi *> (phi_info->stmt);
3715 166835 : gimple *phi_use_stmt = NULL;
3716 166835 : imm_use_iterator imm_iter;
3717 166835 : use_operand_p use_p;
3718 :
3719 : /* When double_reduc is NULL we are testing the inner loop of a
3720 : double reduction. */
3721 166835 : bool inner_loop_of_double_reduc = double_reduc == NULL;
3722 166835 : if (double_reduc)
3723 165728 : *double_reduc = NULL;
3724 166835 : STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION;
3725 :
3726 166835 : tree phi_name = PHI_RESULT (phi);
3727 : /* ??? If there are no uses of the PHI result the inner loop reduction
3728 : won't be detected as possibly double-reduction by vectorizable_reduction
3729 : because that tries to walk the PHI arg from the preheader edge which
3730 : can be constant. See PR60382. */
3731 166835 : if (has_zero_uses (phi_name))
3732 : return NULL;
3733 166699 : class loop *loop = (gimple_bb (phi))->loop_father;
3734 166699 : unsigned nphi_def_loop_uses = 0;
3735 470528 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, phi_name)
3736 : {
3737 315794 : gimple *use_stmt = USE_STMT (use_p);
3738 315794 : if (is_gimple_debug (use_stmt))
3739 90555 : continue;
3740 :
3741 225239 : if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3742 : {
3743 11965 : if (dump_enabled_p ())
3744 35 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3745 : "intermediate value used outside loop.\n");
3746 :
3747 11965 : return NULL;
3748 : }
3749 :
3750 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
3751 : op1 twice (once as definition, once as else) in the same operation.
3752 : Only count it as one. */
3753 213274 : if (use_stmt != phi_use_stmt)
3754 : {
3755 206746 : nphi_def_loop_uses++;
3756 206746 : phi_use_stmt = use_stmt;
3757 : }
3758 11965 : }
3759 :
3760 154734 : tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
3761 154734 : if (TREE_CODE (latch_def) != SSA_NAME)
3762 : {
3763 1489 : if (dump_enabled_p ())
3764 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3765 : "reduction: not ssa_name: %T\n", latch_def);
3766 : return NULL;
3767 : }
3768 :
3769 153245 : stmt_vec_info def_stmt_info = loop_info->lookup_def (latch_def);
3770 153245 : if (!def_stmt_info
3771 153245 : || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt_info->stmt)))
3772 : return NULL;
3773 :
3774 153077 : bool nested_in_vect_loop
3775 153077 : = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info), loop);
3776 153077 : unsigned nlatch_def_loop_uses = 0;
3777 153077 : auto_vec<gphi *, 3> lcphis;
3778 599822 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, latch_def)
3779 : {
3780 446745 : gimple *use_stmt = USE_STMT (use_p);
3781 446745 : if (is_gimple_debug (use_stmt))
3782 137923 : continue;
3783 308822 : if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3784 192913 : nlatch_def_loop_uses++;
3785 : else
3786 : /* We can have more than one loop-closed PHI. */
3787 115909 : lcphis.safe_push (as_a <gphi *> (use_stmt));
3788 153077 : }
3789 :
3790 : /* If we are vectorizing an inner reduction we are executing that
3791 : in the original order only in case we are not dealing with a
3792 : double reduction. */
3793 153077 : if (nested_in_vect_loop && !inner_loop_of_double_reduc)
3794 : {
3795 2433 : if (dump_enabled_p ())
3796 434 : report_vect_op (MSG_NOTE, def_stmt_info->stmt,
3797 : "detected nested cycle: ");
3798 : return def_stmt_info;
3799 : }
3800 :
3801 : /* When the inner loop of a double reduction ends up with more than
3802 : one loop-closed PHI we have failed to classify alternate such
3803 : PHIs as double reduction, leading to wrong code. See PR103237. */
3804 151739 : if (inner_loop_of_double_reduc && lcphis.length () != 1)
3805 : {
3806 1 : if (dump_enabled_p ())
3807 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3808 : "unhandle double reduction\n");
3809 : return NULL;
3810 : }
3811 :
3812 : /* If this isn't a nested cycle or if the nested cycle reduction value
3813 : is used outside of the inner loop we cannot handle uses of the reduction
3814 : value. */
3815 150643 : if (nlatch_def_loop_uses > 1 || nphi_def_loop_uses > 1)
3816 : {
3817 46418 : if (dump_enabled_p ())
3818 406 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3819 : "reduction used in loop.\n");
3820 : return NULL;
3821 : }
3822 :
3823 : /* If DEF_STMT is a phi node itself, we expect it to have a single argument
3824 : defined in the inner loop. */
3825 104225 : if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt))
3826 : {
3827 1366 : tree op1 = PHI_ARG_DEF (def_stmt, 0);
3828 1366 : if (gimple_phi_num_args (def_stmt) != 1
3829 1366 : || TREE_CODE (op1) != SSA_NAME)
3830 : {
3831 95 : if (dump_enabled_p ())
3832 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3833 : "unsupported phi node definition.\n");
3834 :
3835 : return NULL;
3836 : }
3837 :
3838 : /* Verify there is an inner cycle composed of the PHI phi_use_stmt
3839 : and the latch definition op1. */
3840 1271 : gimple *def1 = SSA_NAME_DEF_STMT (op1);
3841 1271 : if (gimple_bb (def1)
3842 1271 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
3843 1271 : && loop->inner
3844 1217 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
3845 1217 : && (is_gimple_assign (def1) || is_gimple_call (def1))
3846 1208 : && is_a <gphi *> (phi_use_stmt)
3847 1196 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt))
3848 1196 : && (op1 == PHI_ARG_DEF_FROM_EDGE (phi_use_stmt,
3849 : loop_latch_edge (loop->inner)))
3850 155455 : && lcphis.length () == 1)
3851 : {
3852 1107 : if (dump_enabled_p ())
3853 144 : report_vect_op (MSG_NOTE, def_stmt,
3854 : "detected double reduction: ");
3855 :
3856 1107 : *double_reduc = as_a <gphi *> (phi_use_stmt);
3857 1107 : return def_stmt_info;
3858 : }
3859 :
3860 : return NULL;
3861 : }
3862 :
3863 : /* Look for the expression computing latch_def from then loop PHI result. */
3864 102859 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3865 102859 : code_helper code;
3866 102859 : if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
3867 : path, inner_loop_of_double_reduc))
3868 : {
3869 98881 : STMT_VINFO_REDUC_CODE (phi_info) = code;
3870 98881 : if (code == COND_EXPR && !nested_in_vect_loop)
3871 8299 : STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION;
3872 :
3873 : /* Fill in STMT_VINFO_REDUC_IDX. */
3874 98881 : unsigned i;
3875 317865 : for (i = path.length () - 1; i >= 1; --i)
3876 : {
3877 120103 : gimple *stmt = USE_STMT (path[i].second);
3878 120103 : stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
3879 120103 : gimple_match_op op;
3880 120103 : if (!gimple_extract_op (stmt, &op))
3881 0 : gcc_unreachable ();
3882 120103 : if (gassign *assign = dyn_cast<gassign *> (stmt))
3883 113886 : STMT_VINFO_REDUC_IDX (stmt_info)
3884 113886 : = path[i].second->use - gimple_assign_rhs1_ptr (assign);
3885 : else
3886 : {
3887 6217 : gcall *call = as_a<gcall *> (stmt);
3888 6217 : STMT_VINFO_REDUC_IDX (stmt_info)
3889 6217 : = path[i].second->use - gimple_call_arg_ptr (call, 0);
3890 : }
3891 : }
3892 98881 : if (dump_enabled_p ())
3893 4134 : dump_printf_loc (MSG_NOTE, vect_location,
3894 : "reduction: detected reduction\n");
3895 :
3896 : return def_stmt_info;
3897 : }
3898 :
3899 3978 : if (dump_enabled_p ())
3900 95 : dump_printf_loc (MSG_NOTE, vect_location,
3901 : "reduction: unknown pattern\n");
3902 :
3903 : return NULL;
3904 255936 : }
3905 :
3906 : /* Estimate the number of peeled epilogue iterations for LOOP_VINFO.
3907 : PEEL_ITERS_PROLOGUE is the number of peeled prologue iterations,
3908 : or -1 if not known. */
3909 :
3910 : static int
3911 506152 : vect_get_peel_iters_epilogue (loop_vec_info loop_vinfo, int peel_iters_prologue)
3912 : {
3913 506152 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
3914 506152 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) || peel_iters_prologue == -1)
3915 : {
3916 211146 : if (dump_enabled_p ())
3917 3641 : dump_printf_loc (MSG_NOTE, vect_location,
3918 : "cost model: epilogue peel iters set to vf/2 "
3919 : "because loop iterations are unknown .\n");
3920 211146 : return assumed_vf / 2;
3921 : }
3922 : else
3923 : {
3924 295006 : int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
3925 295006 : peel_iters_prologue = MIN (niters, peel_iters_prologue);
3926 295006 : int peel_iters_epilogue = (niters - peel_iters_prologue) % assumed_vf;
3927 : /* If we need to peel for gaps, but no peeling is required, we have to
3928 : peel VF iterations. */
3929 295006 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) && !peel_iters_epilogue)
3930 506152 : peel_iters_epilogue = assumed_vf;
3931 : return peel_iters_epilogue;
3932 : }
3933 : }
3934 :
3935 : /* Calculate cost of peeling the scalar loop PEEL_ITERS_PROLOGUE times for
3936 : a prologue and the corresponding times for the epilogue. */
3937 : int
3938 380628 : vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue)
3939 : {
3940 380628 : int retval = 0;
3941 :
3942 380628 : int peel_iters_epilogue
3943 380628 : = vect_get_peel_iters_epilogue (loop_vinfo, peel_iters_prologue);
3944 :
3945 380628 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
3946 : {
3947 : /* If peeled iterations are known but number of scalar loop
3948 : iterations are unknown, count a taken branch per peeled loop. */
3949 144767 : if (peel_iters_prologue > 0)
3950 88970 : retval = builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
3951 144767 : if (peel_iters_epilogue > 0)
3952 144658 : retval += builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
3953 : }
3954 :
3955 761256 : retval += ((peel_iters_prologue + peel_iters_epilogue)
3956 380628 : * loop_vinfo->scalar_costs->body_cost ());
3957 761256 : retval += (((peel_iters_prologue != 0) + (peel_iters_epilogue != 0))
3958 380628 : * loop_vinfo->scalar_costs->outside_cost ());
3959 :
3960 380628 : return retval;
3961 : }
3962 :
3963 : /* Function vect_estimate_min_profitable_iters
3964 :
3965 : Return the number of iterations required for the vector version of the
3966 : loop to be profitable relative to the cost of the scalar version of the
3967 : loop.
3968 :
3969 : *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
3970 : of iterations for vectorization. -1 value means loop vectorization
3971 : is not profitable. This returned value may be used for dynamic
3972 : profitability check.
3973 :
3974 : *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
3975 : for static check against estimated number of iterations. */
3976 :
3977 : static void
3978 142829 : vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
3979 : int *ret_min_profitable_niters,
3980 : int *ret_min_profitable_estimate,
3981 : unsigned *suggested_unroll_factor)
3982 : {
3983 142829 : int min_profitable_iters;
3984 142829 : int min_profitable_estimate;
3985 142829 : int peel_iters_prologue;
3986 142829 : int peel_iters_epilogue;
3987 142829 : unsigned vec_inside_cost = 0;
3988 142829 : int vec_outside_cost = 0;
3989 142829 : unsigned vec_prologue_cost = 0;
3990 142829 : unsigned vec_epilogue_cost = 0;
3991 142829 : int scalar_single_iter_cost = 0;
3992 142829 : int scalar_outside_cost = 0;
3993 142829 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
3994 142829 : int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
3995 142829 : vector_costs *target_cost_data = loop_vinfo->vector_costs;
3996 :
3997 : /* Cost model disabled. */
3998 142829 : if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
3999 : {
4000 16994 : if (dump_enabled_p ())
4001 10678 : dump_printf_loc (MSG_NOTE, vect_location, "cost model disabled.\n");
4002 16994 : *ret_min_profitable_niters = 0;
4003 16994 : *ret_min_profitable_estimate = 0;
4004 16994 : return;
4005 : }
4006 :
4007 : /* Requires loop versioning tests to handle misalignment. */
4008 125835 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
4009 : {
4010 : /* FIXME: Make cost depend on complexity of individual check. */
4011 18 : unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
4012 18 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4013 18 : if (dump_enabled_p ())
4014 2 : dump_printf (MSG_NOTE,
4015 : "cost model: Adding cost of checks for loop "
4016 : "versioning to treat misalignment.\n");
4017 : }
4018 :
4019 : /* Requires loop versioning with alias checks. */
4020 125835 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
4021 : {
4022 : /* FIXME: Make cost depend on complexity of individual check. */
4023 7201 : unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
4024 7201 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4025 7201 : len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
4026 4 : if (len)
4027 : /* Count LEN - 1 ANDs and LEN comparisons. */
4028 4 : (void) add_stmt_cost (target_cost_data, len * 2 - 1,
4029 : scalar_stmt, vect_prologue);
4030 7201 : len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
4031 1259 : if (len)
4032 : {
4033 : /* Count LEN - 1 ANDs and LEN comparisons. */
4034 1259 : unsigned int nstmts = len * 2 - 1;
4035 : /* +1 for each bias that needs adding. */
4036 2518 : for (unsigned int i = 0; i < len; ++i)
4037 1259 : if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
4038 154 : nstmts += 1;
4039 1259 : (void) add_stmt_cost (target_cost_data, nstmts,
4040 : scalar_stmt, vect_prologue);
4041 : }
4042 7201 : if (dump_enabled_p ())
4043 32 : dump_printf (MSG_NOTE,
4044 : "cost model: Adding cost of checks for loop "
4045 : "versioning aliasing.\n");
4046 : }
4047 :
4048 : /* Requires loop versioning with niter checks. */
4049 125835 : if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
4050 : {
4051 : /* FIXME: Make cost depend on complexity of individual check. */
4052 749 : (void) add_stmt_cost (target_cost_data, 1, vector_stmt,
4053 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4054 749 : if (dump_enabled_p ())
4055 1 : dump_printf (MSG_NOTE,
4056 : "cost model: Adding cost of checks for loop "
4057 : "versioning niters.\n");
4058 : }
4059 :
4060 125835 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4061 7962 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4062 : vect_prologue);
4063 :
4064 : /* Count statements in scalar loop. Using this as scalar cost for a single
4065 : iteration for now.
4066 :
4067 : TODO: Add outer loop support.
4068 :
4069 : TODO: Consider assigning different costs to different scalar
4070 : statements. */
4071 :
4072 125835 : scalar_single_iter_cost = loop_vinfo->scalar_costs->total_cost ();
4073 :
4074 : /* Add additional cost for the peeled instructions in prologue and epilogue
4075 : loop. (For fully-masked loops there will be no peeling.)
4076 :
4077 : FORNOW: If we don't know the value of peel_iters for prologue or epilogue
4078 : at compile-time - we assume it's vf/2 (the worst would be vf-1).
4079 :
4080 : TODO: Build an expression that represents peel_iters for prologue and
4081 : epilogue to be used in a run-time test. */
4082 :
4083 125835 : bool prologue_need_br_taken_cost = false;
4084 125835 : bool prologue_need_br_not_taken_cost = false;
4085 :
4086 : /* Calculate peel_iters_prologue. */
4087 125835 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
4088 : peel_iters_prologue = 0;
4089 125835 : else if (npeel < 0)
4090 : {
4091 282 : peel_iters_prologue = assumed_vf / 2;
4092 282 : if (dump_enabled_p ())
4093 11 : dump_printf (MSG_NOTE, "cost model: "
4094 : "prologue peel iters set to vf/2.\n");
4095 :
4096 : /* If peeled iterations are unknown, count a taken branch and a not taken
4097 : branch per peeled loop. Even if scalar loop iterations are known,
4098 : vector iterations are not known since peeled prologue iterations are
4099 : not known. Hence guards remain the same. */
4100 : prologue_need_br_taken_cost = true;
4101 : prologue_need_br_not_taken_cost = true;
4102 : }
4103 : else
4104 : {
4105 125553 : peel_iters_prologue = npeel;
4106 125553 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_prologue > 0)
4107 : /* If peeled iterations are known but number of scalar loop
4108 : iterations are unknown, count a taken branch per peeled loop. */
4109 125835 : prologue_need_br_taken_cost = true;
4110 : }
4111 :
4112 125835 : bool epilogue_need_br_taken_cost = false;
4113 125835 : bool epilogue_need_br_not_taken_cost = false;
4114 :
4115 : /* Calculate peel_iters_epilogue. */
4116 125835 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4117 : /* We need to peel exactly one iteration for gaps. */
4118 29 : peel_iters_epilogue = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
4119 125806 : else if (npeel < 0)
4120 : {
4121 : /* If peeling for alignment is unknown, loop bound of main loop
4122 : becomes unknown. */
4123 282 : peel_iters_epilogue = assumed_vf / 2;
4124 282 : if (dump_enabled_p ())
4125 11 : dump_printf (MSG_NOTE, "cost model: "
4126 : "epilogue peel iters set to vf/2 because "
4127 : "peeling for alignment is unknown.\n");
4128 :
4129 : /* See the same reason above in peel_iters_prologue calculation. */
4130 : epilogue_need_br_taken_cost = true;
4131 : epilogue_need_br_not_taken_cost = true;
4132 : }
4133 : else
4134 : {
4135 125524 : peel_iters_epilogue = vect_get_peel_iters_epilogue (loop_vinfo, npeel);
4136 125524 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_epilogue > 0)
4137 : /* If peeled iterations are known but number of scalar loop
4138 : iterations are unknown, count a taken branch per peeled loop. */
4139 125835 : epilogue_need_br_taken_cost = true;
4140 : }
4141 :
4142 : /* The way we cummulate peeling costs into the vector prologue/epilogue
4143 : cost is a bit awkward given we cannot reuse scalar_costs which is
4144 : already computed and also because it cannot take into account any
4145 : epilogue vectorization we'll carry out in the end. */
4146 :
4147 125835 : stmt_info_for_cost *si;
4148 125835 : int j;
4149 : /* Add costs associated with peel_iters_prologue. */
4150 125835 : if (peel_iters_prologue)
4151 1083 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4152 : {
4153 785 : (void) add_stmt_cost (target_cost_data,
4154 785 : si->count * peel_iters_prologue, si->kind,
4155 : si->stmt_info, si->node, si->vectype,
4156 : si->misalign, vect_prologue);
4157 : }
4158 :
4159 : /* Add costs associated with peel_iters_epilogue. */
4160 125835 : if (peel_iters_epilogue)
4161 390609 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4162 : {
4163 312735 : (void) add_stmt_cost (target_cost_data,
4164 312735 : si->count * peel_iters_epilogue, si->kind,
4165 : si->stmt_info, si->node, si->vectype,
4166 : si->misalign, vect_epilogue);
4167 : }
4168 :
4169 : /* Add possible cond_branch_taken/cond_branch_not_taken cost. */
4170 :
4171 125835 : if (prologue_need_br_taken_cost)
4172 282 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4173 : vect_prologue);
4174 :
4175 125835 : if (prologue_need_br_not_taken_cost)
4176 282 : (void) add_stmt_cost (target_cost_data, 1,
4177 : cond_branch_not_taken, vect_prologue);
4178 :
4179 125835 : if (epilogue_need_br_taken_cost)
4180 65792 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4181 : vect_epilogue);
4182 :
4183 125835 : if (epilogue_need_br_not_taken_cost)
4184 282 : (void) add_stmt_cost (target_cost_data, 1,
4185 : cond_branch_not_taken, vect_epilogue);
4186 :
4187 : /* Take care of special costs for rgroup controls of partial vectors. */
4188 29 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4189 125864 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4190 : == vect_partial_vectors_avx512))
4191 : {
4192 : /* Calculate how many masks we need to generate. */
4193 29 : unsigned int num_masks = 0;
4194 29 : bool need_saturation = false;
4195 121 : for (auto rgm : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
4196 34 : if (rgm.type)
4197 : {
4198 29 : unsigned nvectors = rgm.factor;
4199 29 : num_masks += nvectors;
4200 29 : if (TYPE_PRECISION (TREE_TYPE (rgm.compare_type))
4201 29 : < TYPE_PRECISION (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo)))
4202 10 : need_saturation = true;
4203 : }
4204 :
4205 : /* ??? The target isn't able to identify the costs below as
4206 : producing masks so it cannot penaltize cases where we'd run
4207 : out of mask registers for example. */
4208 :
4209 : /* ??? We are also failing to account for smaller vector masks
4210 : we generate by splitting larger masks in vect_get_loop_mask. */
4211 :
4212 : /* In the worst case, we need to generate each mask in the prologue
4213 : and in the loop body. We need one splat per group and one
4214 : compare per mask.
4215 :
4216 : Sometimes the prologue mask will fold to a constant,
4217 : so the actual prologue cost might be smaller. However, it's
4218 : simpler and safer to use the worst-case cost; if this ends up
4219 : being the tie-breaker between vectorizing or not, then it's
4220 : probably better not to vectorize. */
4221 29 : (void) add_stmt_cost (target_cost_data,
4222 : num_masks
4223 29 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4224 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4225 : vect_prologue);
4226 58 : (void) add_stmt_cost (target_cost_data,
4227 : num_masks
4228 58 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4229 : vector_stmt, NULL, NULL, NULL_TREE, 0, vect_body);
4230 :
4231 : /* When we need saturation we need it both in the prologue and
4232 : the epilogue. */
4233 29 : if (need_saturation)
4234 : {
4235 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4236 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4237 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4238 : NULL, NULL, NULL_TREE, 0, vect_body);
4239 : }
4240 : }
4241 0 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4242 125806 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4243 : == vect_partial_vectors_while_ult))
4244 : {
4245 : /* Calculate how many masks we need to generate. */
4246 : unsigned int num_masks = 0;
4247 : rgroup_controls *rgm;
4248 : unsigned int num_vectors_m1;
4249 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec,
4250 : num_vectors_m1, rgm)
4251 0 : if (rgm->type)
4252 0 : num_masks += num_vectors_m1 + 1;
4253 0 : gcc_assert (num_masks > 0);
4254 :
4255 : /* In the worst case, we need to generate each mask in the prologue
4256 : and in the loop body. One of the loop body mask instructions
4257 : replaces the comparison in the scalar loop, and since we don't
4258 : count the scalar comparison against the scalar body, we shouldn't
4259 : count that vector instruction against the vector body either.
4260 :
4261 : Sometimes we can use unpacks instead of generating prologue
4262 : masks and sometimes the prologue mask will fold to a constant,
4263 : so the actual prologue cost might be smaller. However, it's
4264 : simpler and safer to use the worst-case cost; if this ends up
4265 : being the tie-breaker between vectorizing or not, then it's
4266 : probably better not to vectorize. */
4267 0 : (void) add_stmt_cost (target_cost_data, num_masks,
4268 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4269 : vect_prologue);
4270 0 : (void) add_stmt_cost (target_cost_data, num_masks - 1,
4271 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4272 : vect_body);
4273 : }
4274 125806 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
4275 : {
4276 : /* Referring to the functions vect_set_loop_condition_partial_vectors
4277 : and vect_set_loop_controls_directly, we need to generate each
4278 : length in the prologue and in the loop body if required. Although
4279 : there are some possible optimizations, we consider the worst case
4280 : here. */
4281 :
4282 0 : bool niters_known_p = LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo);
4283 0 : signed char partial_load_store_bias
4284 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
4285 0 : bool need_iterate_p
4286 0 : = (!LOOP_VINFO_EPILOGUE_P (loop_vinfo)
4287 0 : && !vect_known_niters_smaller_than_vf (loop_vinfo));
4288 :
4289 : /* Calculate how many statements to be added. */
4290 0 : unsigned int prologue_stmts = 0;
4291 0 : unsigned int body_stmts = 0;
4292 :
4293 0 : rgroup_controls *rgc;
4294 0 : unsigned int num_vectors_m1;
4295 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), num_vectors_m1, rgc)
4296 0 : if (rgc->type)
4297 : {
4298 : /* May need one SHIFT for nitems_total computation. */
4299 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
4300 0 : if (nitems != 1 && !niters_known_p)
4301 0 : prologue_stmts += 1;
4302 :
4303 : /* May need one MAX and one MINUS for wrap around. */
4304 0 : if (vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc))
4305 0 : prologue_stmts += 2;
4306 :
4307 : /* Need one MAX and one MINUS for each batch limit excepting for
4308 : the 1st one. */
4309 0 : prologue_stmts += num_vectors_m1 * 2;
4310 :
4311 0 : unsigned int num_vectors = num_vectors_m1 + 1;
4312 :
4313 : /* Need to set up lengths in prologue, only one MIN required
4314 : for each since start index is zero. */
4315 0 : prologue_stmts += num_vectors;
4316 :
4317 : /* If we have a non-zero partial load bias, we need one PLUS
4318 : to adjust the load length. */
4319 0 : if (partial_load_store_bias != 0)
4320 0 : body_stmts += 1;
4321 :
4322 0 : unsigned int length_update_cost = 0;
4323 0 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
4324 : /* For decrement IV style, Each only need a single SELECT_VL
4325 : or MIN since beginning to calculate the number of elements
4326 : need to be processed in current iteration. */
4327 : length_update_cost = 1;
4328 : else
4329 : /* For increment IV stype, Each may need two MINs and one MINUS to
4330 : update lengths in body for next iteration. */
4331 0 : length_update_cost = 3;
4332 :
4333 0 : if (need_iterate_p)
4334 0 : body_stmts += length_update_cost * num_vectors;
4335 : }
4336 :
4337 0 : (void) add_stmt_cost (target_cost_data, prologue_stmts,
4338 : scalar_stmt, vect_prologue);
4339 0 : (void) add_stmt_cost (target_cost_data, body_stmts,
4340 : scalar_stmt, vect_body);
4341 : }
4342 :
4343 : /* FORNOW: The scalar outside cost is incremented in one of the
4344 : following ways:
4345 :
4346 : 1. The vectorizer checks for alignment and aliasing and generates
4347 : a condition that allows dynamic vectorization. A cost model
4348 : check is ANDED with the versioning condition. Hence scalar code
4349 : path now has the added cost of the versioning check.
4350 :
4351 : if (cost > th & versioning_check)
4352 : jmp to vector code
4353 :
4354 : Hence run-time scalar is incremented by not-taken branch cost.
4355 :
4356 : 2. The vectorizer then checks if a prologue is required. If the
4357 : cost model check was not done before during versioning, it has to
4358 : be done before the prologue check.
4359 :
4360 : if (cost <= th)
4361 : prologue = scalar_iters
4362 : if (prologue == 0)
4363 : jmp to vector code
4364 : else
4365 : execute prologue
4366 : if (prologue == num_iters)
4367 : go to exit
4368 :
4369 : Hence the run-time scalar cost is incremented by a taken branch,
4370 : plus a not-taken branch, plus a taken branch cost.
4371 :
4372 : 3. The vectorizer then checks if an epilogue is required. If the
4373 : cost model check was not done before during prologue check, it
4374 : has to be done with the epilogue check.
4375 :
4376 : if (prologue == 0)
4377 : jmp to vector code
4378 : else
4379 : execute prologue
4380 : if (prologue == num_iters)
4381 : go to exit
4382 : vector code:
4383 : if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
4384 : jmp to epilogue
4385 :
4386 : Hence the run-time scalar cost should be incremented by 2 taken
4387 : branches.
4388 :
4389 : TODO: The back end may reorder the BBS's differently and reverse
4390 : conditions/branch directions. Change the estimates below to
4391 : something more reasonable. */
4392 :
4393 : /* If the number of iterations is known and we do not do versioning, we can
4394 : decide whether to vectorize at compile time. Hence the scalar version
4395 : do not carry cost model guard costs. */
4396 59194 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
4397 185029 : || LOOP_REQUIRES_VERSIONING (loop_vinfo))
4398 : {
4399 : /* Cost model check occurs at versioning. */
4400 67783 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4401 7962 : scalar_outside_cost += vect_get_stmt_cost (cond_branch_not_taken);
4402 : else
4403 : {
4404 : /* Cost model check occurs at prologue generation. */
4405 59821 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
4406 155 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken)
4407 155 : + vect_get_stmt_cost (cond_branch_not_taken);
4408 : /* Cost model check occurs at epilogue generation. */
4409 : else
4410 59666 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken);
4411 : }
4412 : }
4413 :
4414 : /* Complete the target-specific cost calculations. */
4415 125835 : loop_vinfo->vector_costs->finish_cost (loop_vinfo->scalar_costs);
4416 125835 : vec_prologue_cost = loop_vinfo->vector_costs->prologue_cost ();
4417 125835 : vec_inside_cost = loop_vinfo->vector_costs->body_cost ();
4418 125835 : vec_epilogue_cost = loop_vinfo->vector_costs->epilogue_cost ();
4419 125835 : if (suggested_unroll_factor)
4420 125445 : *suggested_unroll_factor
4421 125445 : = loop_vinfo->vector_costs->suggested_unroll_factor ();
4422 :
4423 125445 : if (suggested_unroll_factor && *suggested_unroll_factor > 1
4424 419 : && LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) != MAX_VECTORIZATION_FACTOR
4425 0 : && !known_le (LOOP_VINFO_VECT_FACTOR (loop_vinfo) *
4426 : *suggested_unroll_factor,
4427 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo)))
4428 : {
4429 0 : if (dump_enabled_p ())
4430 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4431 : "can't unroll as unrolled vectorization factor larger"
4432 : " than maximum vectorization factor: "
4433 : HOST_WIDE_INT_PRINT_UNSIGNED "\n",
4434 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo));
4435 0 : *suggested_unroll_factor = 1;
4436 : }
4437 :
4438 125835 : vec_outside_cost = (int)(vec_prologue_cost + vec_epilogue_cost);
4439 :
4440 125835 : if (dump_enabled_p ())
4441 : {
4442 1094 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
4443 1094 : dump_printf (MSG_NOTE, " Vector inside of loop cost: %d\n",
4444 : vec_inside_cost);
4445 1094 : dump_printf (MSG_NOTE, " Vector prologue cost: %d\n",
4446 : vec_prologue_cost);
4447 1094 : dump_printf (MSG_NOTE, " Vector epilogue cost: %d\n",
4448 : vec_epilogue_cost);
4449 1094 : dump_printf (MSG_NOTE, " Scalar iteration cost: %d\n",
4450 : scalar_single_iter_cost);
4451 1094 : dump_printf (MSG_NOTE, " Scalar outside cost: %d\n",
4452 : scalar_outside_cost);
4453 1094 : dump_printf (MSG_NOTE, " Vector outside cost: %d\n",
4454 : vec_outside_cost);
4455 1094 : dump_printf (MSG_NOTE, " prologue iterations: %d\n",
4456 : peel_iters_prologue);
4457 1094 : dump_printf (MSG_NOTE, " epilogue iterations: %d\n",
4458 : peel_iters_epilogue);
4459 : }
4460 :
4461 : /* Calculate number of iterations required to make the vector version
4462 : profitable, relative to the loop bodies only. The following condition
4463 : must hold true:
4464 : SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
4465 : where
4466 : SIC = scalar iteration cost, VIC = vector iteration cost,
4467 : VOC = vector outside cost, VF = vectorization factor,
4468 : NPEEL = prologue iterations + epilogue iterations,
4469 : SOC = scalar outside cost for run time cost model check. */
4470 :
4471 125835 : int saving_per_viter = (scalar_single_iter_cost * assumed_vf
4472 125835 : - vec_inside_cost);
4473 125835 : if (saving_per_viter <= 0)
4474 : {
4475 24415 : if (LOOP_VINFO_LOOP (loop_vinfo)->force_vectorize)
4476 0 : warning_at (vect_location.get_location_t (), OPT_Wopenmp_simd,
4477 : "vectorization did not happen for a simd loop");
4478 :
4479 24415 : if (dump_enabled_p ())
4480 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4481 : "cost model: the vector iteration cost = %d "
4482 : "divided by the scalar iteration cost = %d "
4483 : "is greater or equal to the vectorization factor = %d"
4484 : ".\n",
4485 : vec_inside_cost, scalar_single_iter_cost, assumed_vf);
4486 24415 : *ret_min_profitable_niters = -1;
4487 24415 : *ret_min_profitable_estimate = -1;
4488 24415 : return;
4489 : }
4490 :
4491 : /* ??? The "if" arm is written to handle all cases; see below for what
4492 : we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4493 101420 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4494 : {
4495 : /* Rewriting the condition above in terms of the number of
4496 : vector iterations (vniters) rather than the number of
4497 : scalar iterations (niters) gives:
4498 :
4499 : SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
4500 :
4501 : <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
4502 :
4503 : For integer N, X and Y when X > 0:
4504 :
4505 : N * X > Y <==> N >= (Y /[floor] X) + 1. */
4506 21 : int outside_overhead = (vec_outside_cost
4507 21 : - scalar_single_iter_cost * peel_iters_prologue
4508 21 : - scalar_single_iter_cost * peel_iters_epilogue
4509 : - scalar_outside_cost);
4510 : /* We're only interested in cases that require at least one
4511 : vector iteration. */
4512 21 : int min_vec_niters = 1;
4513 21 : if (outside_overhead > 0)
4514 16 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4515 :
4516 21 : if (dump_enabled_p ())
4517 8 : dump_printf (MSG_NOTE, " Minimum number of vector iterations: %d\n",
4518 : min_vec_niters);
4519 :
4520 21 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4521 : {
4522 : /* Now that we know the minimum number of vector iterations,
4523 : find the minimum niters for which the scalar cost is larger:
4524 :
4525 : SIC * niters > VIC * vniters + VOC - SOC
4526 :
4527 : We know that the minimum niters is no more than
4528 : vniters * VF + NPEEL, but it might be (and often is) less
4529 : than that if a partial vector iteration is cheaper than the
4530 : equivalent scalar code. */
4531 21 : int threshold = (vec_inside_cost * min_vec_niters
4532 21 : + vec_outside_cost
4533 21 : - scalar_outside_cost);
4534 21 : if (threshold <= 0)
4535 : min_profitable_iters = 1;
4536 : else
4537 21 : min_profitable_iters = threshold / scalar_single_iter_cost + 1;
4538 : }
4539 : else
4540 : /* Convert the number of vector iterations into a number of
4541 : scalar iterations. */
4542 0 : min_profitable_iters = (min_vec_niters * assumed_vf
4543 0 : + peel_iters_prologue
4544 : + peel_iters_epilogue);
4545 : }
4546 : else
4547 : {
4548 101399 : min_profitable_iters = ((vec_outside_cost - scalar_outside_cost)
4549 101399 : * assumed_vf
4550 101399 : - vec_inside_cost * peel_iters_prologue
4551 101399 : - vec_inside_cost * peel_iters_epilogue);
4552 101399 : if (min_profitable_iters <= 0)
4553 : min_profitable_iters = 0;
4554 : else
4555 : {
4556 86286 : min_profitable_iters /= saving_per_viter;
4557 :
4558 86286 : if ((scalar_single_iter_cost * assumed_vf * min_profitable_iters)
4559 86286 : <= (((int) vec_inside_cost * min_profitable_iters)
4560 86286 : + (((int) vec_outside_cost - scalar_outside_cost)
4561 : * assumed_vf)))
4562 86286 : min_profitable_iters++;
4563 : }
4564 : }
4565 :
4566 101420 : if (dump_enabled_p ())
4567 1064 : dump_printf (MSG_NOTE,
4568 : " Calculated minimum iters for profitability: %d\n",
4569 : min_profitable_iters);
4570 :
4571 101420 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
4572 101399 : && min_profitable_iters < (assumed_vf + peel_iters_prologue))
4573 : /* We want the vectorized loop to execute at least once. */
4574 : min_profitable_iters = assumed_vf + peel_iters_prologue;
4575 22228 : else if (min_profitable_iters < peel_iters_prologue)
4576 : /* For LOOP_VINFO_USING_PARTIAL_VECTORS_P, we need to ensure the
4577 : vectorized loop executes at least once. */
4578 : min_profitable_iters = peel_iters_prologue;
4579 :
4580 101420 : if (dump_enabled_p ())
4581 1064 : dump_printf_loc (MSG_NOTE, vect_location,
4582 : " Runtime profitability threshold = %d\n",
4583 : min_profitable_iters);
4584 :
4585 101420 : *ret_min_profitable_niters = min_profitable_iters;
4586 :
4587 : /* Calculate number of iterations required to make the vector version
4588 : profitable, relative to the loop bodies only.
4589 :
4590 : Non-vectorized variant is SIC * niters and it must win over vector
4591 : variant on the expected loop trip count. The following condition must hold true:
4592 : SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC */
4593 :
4594 101420 : if (vec_outside_cost <= 0)
4595 : min_profitable_estimate = 0;
4596 : /* ??? This "else if" arm is written to handle all cases; see below for
4597 : what we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4598 90791 : else if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4599 : {
4600 : /* This is a repeat of the code above, but with + SOC rather
4601 : than - SOC. */
4602 21 : int outside_overhead = (vec_outside_cost
4603 21 : - scalar_single_iter_cost * peel_iters_prologue
4604 21 : - scalar_single_iter_cost * peel_iters_epilogue
4605 : + scalar_outside_cost);
4606 21 : int min_vec_niters = 1;
4607 21 : if (outside_overhead > 0)
4608 21 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4609 :
4610 21 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4611 : {
4612 21 : int threshold = (vec_inside_cost * min_vec_niters
4613 21 : + vec_outside_cost
4614 21 : + scalar_outside_cost);
4615 21 : min_profitable_estimate = threshold / scalar_single_iter_cost + 1;
4616 : }
4617 : else
4618 : min_profitable_estimate = (min_vec_niters * assumed_vf
4619 : + peel_iters_prologue
4620 : + peel_iters_epilogue);
4621 : }
4622 : else
4623 : {
4624 90770 : min_profitable_estimate = ((vec_outside_cost + scalar_outside_cost)
4625 90770 : * assumed_vf
4626 90770 : - vec_inside_cost * peel_iters_prologue
4627 90770 : - vec_inside_cost * peel_iters_epilogue)
4628 90770 : / ((scalar_single_iter_cost * assumed_vf)
4629 : - vec_inside_cost);
4630 : }
4631 101420 : min_profitable_estimate = MAX (min_profitable_estimate, min_profitable_iters);
4632 101420 : if (dump_enabled_p ())
4633 1064 : dump_printf_loc (MSG_NOTE, vect_location,
4634 : " Static estimate profitability threshold = %d\n",
4635 : min_profitable_estimate);
4636 :
4637 101420 : *ret_min_profitable_estimate = min_profitable_estimate;
4638 : }
4639 :
4640 : /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
4641 : vector elements (not bits) for a vector with NELT elements. */
4642 : static void
4643 2285 : calc_vec_perm_mask_for_shift (unsigned int offset, unsigned int nelt,
4644 : vec_perm_builder *sel)
4645 : {
4646 : /* The encoding is a single stepped pattern. Any wrap-around is handled
4647 : by vec_perm_indices. */
4648 2285 : sel->new_vector (nelt, 1, 3);
4649 9140 : for (unsigned int i = 0; i < 3; i++)
4650 6855 : sel->quick_push (i + offset);
4651 2285 : }
4652 :
4653 : /* Checks whether the target supports whole-vector shifts for vectors of mode
4654 : MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
4655 : it supports vec_perm_const with masks for all necessary shift amounts. */
4656 : static bool
4657 13856 : have_whole_vector_shift (machine_mode mode)
4658 : {
4659 13856 : if (can_implement_p (vec_shr_optab, mode))
4660 : return true;
4661 :
4662 : /* Variable-length vectors should be handled via the optab. */
4663 63 : unsigned int nelt;
4664 126 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
4665 : return false;
4666 :
4667 63 : vec_perm_builder sel;
4668 63 : vec_perm_indices indices;
4669 315 : for (unsigned int i = nelt / 2; i >= 1; i /= 2)
4670 : {
4671 252 : calc_vec_perm_mask_for_shift (i, nelt, &sel);
4672 252 : indices.new_vector (sel, 2, nelt);
4673 252 : if (!can_vec_perm_const_p (mode, mode, indices, false))
4674 : return false;
4675 : }
4676 : return true;
4677 63 : }
4678 :
4679 : /* Return true if (a) STMT_INFO is a DOT_PROD_EXPR reduction whose
4680 : multiplication operands have differing signs and (b) we intend
4681 : to emulate the operation using a series of signed DOT_PROD_EXPRs.
4682 : See vect_emulate_mixed_dot_prod for the actual sequence used. */
4683 :
4684 : static bool
4685 2476 : vect_is_emulated_mixed_dot_prod (slp_tree slp_node)
4686 : {
4687 2476 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_node);
4688 2476 : gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
4689 2023 : if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
4690 : return false;
4691 :
4692 832 : tree rhs1 = gimple_assign_rhs1 (assign);
4693 832 : tree rhs2 = gimple_assign_rhs2 (assign);
4694 832 : if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
4695 : return false;
4696 :
4697 627 : return !directly_supported_p (DOT_PROD_EXPR,
4698 : SLP_TREE_VECTYPE (slp_node),
4699 209 : SLP_TREE_VECTYPE
4700 : (SLP_TREE_CHILDREN (slp_node)[0]),
4701 209 : optab_vector_mixed_sign);
4702 : }
4703 :
4704 : /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
4705 : functions. Design better to avoid maintenance issues. */
4706 :
4707 : /* Function vect_model_reduction_cost.
4708 :
4709 : Models cost for a reduction operation, including the vector ops
4710 : generated within the strip-mine loop in some cases, the initial
4711 : definition before the loop, and the epilogue code that must be generated. */
4712 :
4713 : static void
4714 72611 : vect_model_reduction_cost (loop_vec_info loop_vinfo,
4715 : slp_tree node, internal_fn reduc_fn,
4716 : vect_reduction_type reduction_type,
4717 : int ncopies, stmt_vector_for_cost *cost_vec)
4718 : {
4719 72611 : int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
4720 72611 : tree vectype;
4721 72611 : machine_mode mode;
4722 72611 : class loop *loop = NULL;
4723 :
4724 72611 : if (loop_vinfo)
4725 72611 : loop = LOOP_VINFO_LOOP (loop_vinfo);
4726 :
4727 : /* Condition reductions generate two reductions in the loop. */
4728 72611 : if (reduction_type == COND_REDUCTION)
4729 324 : ncopies *= 2;
4730 :
4731 72611 : vectype = SLP_TREE_VECTYPE (node);
4732 72611 : mode = TYPE_MODE (vectype);
4733 72611 : stmt_vec_info orig_stmt_info
4734 72611 : = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
4735 :
4736 72611 : gimple_match_op op;
4737 72611 : if (!gimple_extract_op (orig_stmt_info->stmt, &op))
4738 0 : gcc_unreachable ();
4739 :
4740 72611 : if (reduction_type == EXTRACT_LAST_REDUCTION)
4741 : /* No extra instructions are needed in the prologue. The loop body
4742 : operations are costed in vectorizable_condition. */
4743 : inside_cost = 0;
4744 72611 : else if (reduction_type == FOLD_LEFT_REDUCTION)
4745 : {
4746 : /* No extra instructions needed in the prologue. */
4747 4301 : prologue_cost = 0;
4748 :
4749 4301 : if (reduc_fn != IFN_LAST)
4750 : /* Count one reduction-like operation per vector. */
4751 0 : inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
4752 : node, 0, vect_body);
4753 : else
4754 : {
4755 : /* Use NCOPIES deconstructs and NELEMENTS scalar ops. */
4756 4301 : unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
4757 4301 : inside_cost = record_stmt_cost (cost_vec, ncopies,
4758 : vec_deconstruct, node, 0,
4759 : vect_body);
4760 4301 : inside_cost += record_stmt_cost (cost_vec, nelements,
4761 : scalar_stmt, node, 0,
4762 : vect_body);
4763 : }
4764 : }
4765 : else
4766 : {
4767 : /* Add in the cost of the initial definitions. */
4768 68310 : int prologue_stmts;
4769 68310 : if (reduction_type == COND_REDUCTION)
4770 : /* For cond reductions we have four vectors: initial index, step,
4771 : initial result of the data reduction, initial value of the index
4772 : reduction. */
4773 : prologue_stmts = 4;
4774 : else
4775 : /* We need the initial reduction value. */
4776 67986 : prologue_stmts = 1;
4777 68310 : prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
4778 : scalar_to_vec, node, 0,
4779 : vect_prologue);
4780 : }
4781 :
4782 : /* Determine cost of epilogue code.
4783 :
4784 : We have a reduction operator that will reduce the vector in one statement.
4785 : Also requires scalar extract. */
4786 :
4787 72611 : if (!loop || !nested_in_vect_loop_p (loop, orig_stmt_info))
4788 : {
4789 72427 : if (reduc_fn != IFN_LAST)
4790 : {
4791 52649 : if (reduction_type == COND_REDUCTION)
4792 : {
4793 : /* An EQ stmt and an COND_EXPR stmt. */
4794 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4795 : vector_stmt, node, 0,
4796 : vect_epilogue);
4797 : /* Reduction of the max index and a reduction of the found
4798 : values. */
4799 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4800 : vec_to_scalar, node, 0,
4801 : vect_epilogue);
4802 : /* A broadcast of the max value. */
4803 18 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4804 : scalar_to_vec, node, 0,
4805 : vect_epilogue);
4806 : }
4807 : else
4808 : {
4809 52631 : epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
4810 : node, 0, vect_epilogue);
4811 52631 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4812 : vec_to_scalar, node, 0,
4813 : vect_epilogue);
4814 : }
4815 : }
4816 19778 : else if (reduction_type == COND_REDUCTION)
4817 : {
4818 306 : unsigned estimated_nunits = vect_nunits_for_cost (vectype);
4819 : /* Extraction of scalar elements. */
4820 306 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4821 : vec_deconstruct, node, 0,
4822 : vect_epilogue);
4823 : /* Scalar max reductions via COND_EXPR / MAX_EXPR. */
4824 306 : epilogue_cost += record_stmt_cost (cost_vec,
4825 306 : 2 * estimated_nunits - 3,
4826 : scalar_stmt, node, 0,
4827 : vect_epilogue);
4828 : }
4829 19472 : else if (reduction_type == EXTRACT_LAST_REDUCTION
4830 19472 : || reduction_type == FOLD_LEFT_REDUCTION)
4831 : /* No extra instructions need in the epilogue. */
4832 : ;
4833 : else
4834 : {
4835 15171 : int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
4836 15171 : tree bitsize = TYPE_SIZE (op.type);
4837 15171 : int element_bitsize = tree_to_uhwi (bitsize);
4838 15171 : int nelements = vec_size_in_bits / element_bitsize;
4839 :
4840 15171 : if (op.code == COND_EXPR)
4841 31 : op.code = MAX_EXPR;
4842 :
4843 : /* We have a whole vector shift available. */
4844 3141 : if (VECTOR_MODE_P (mode)
4845 15171 : && directly_supported_p (op.code, vectype)
4846 27063 : && have_whole_vector_shift (mode))
4847 : {
4848 : /* Final reduction via vector shifts and the reduction operator.
4849 : Also requires scalar extract. */
4850 35676 : epilogue_cost += record_stmt_cost (cost_vec,
4851 23784 : exact_log2 (nelements) * 2,
4852 : vector_stmt, node, 0,
4853 : vect_epilogue);
4854 11892 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4855 : vec_to_scalar, node, 0,
4856 : vect_epilogue);
4857 : }
4858 : else
4859 : /* Use extracts and reduction op for final reduction. For N
4860 : elements, we have N extracts and N-1 reduction ops. */
4861 3279 : epilogue_cost += record_stmt_cost (cost_vec,
4862 3279 : nelements + nelements - 1,
4863 : vector_stmt, node, 0,
4864 : vect_epilogue);
4865 : }
4866 : }
4867 :
4868 72611 : if (dump_enabled_p ())
4869 3007 : dump_printf (MSG_NOTE,
4870 : "vect_model_reduction_cost: inside_cost = %d, "
4871 : "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost,
4872 : prologue_cost, epilogue_cost);
4873 72611 : }
4874 :
4875 : /* SEQ is a sequence of instructions that initialize the reduction
4876 : described by REDUC_INFO. Emit them in the appropriate place. */
4877 :
4878 : static void
4879 455 : vect_emit_reduction_init_stmts (loop_vec_info loop_vinfo,
4880 : vect_reduc_info reduc_info, gimple *seq)
4881 : {
4882 455 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
4883 : {
4884 : /* When reusing an accumulator from the main loop, we only need
4885 : initialization instructions if the main loop can be skipped.
4886 : In that case, emit the initialization instructions at the end
4887 : of the guard block that does the skip. */
4888 18 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
4889 18 : gcc_assert (skip_edge);
4890 18 : gimple_stmt_iterator gsi = gsi_last_bb (skip_edge->src);
4891 18 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
4892 : }
4893 : else
4894 : {
4895 : /* The normal case: emit the initialization instructions on the
4896 : preheader edge. */
4897 437 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
4898 437 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), seq);
4899 : }
4900 455 : }
4901 :
4902 : /* Get at the initial defs for the reduction PHIs for REDUC_INFO,
4903 : which performs a reduction involving GROUP_SIZE scalar statements.
4904 : NUMBER_OF_VECTORS is the number of vector defs to create. If NEUTRAL_OP
4905 : is nonnull, introducing extra elements of that value will not change the
4906 : result. */
4907 :
4908 : static void
4909 21829 : get_initial_defs_for_reduction (loop_vec_info loop_vinfo,
4910 : vect_reduc_info reduc_info,
4911 : tree vector_type,
4912 : vec<tree> *vec_oprnds,
4913 : unsigned int number_of_vectors,
4914 : unsigned int group_size, tree neutral_op)
4915 : {
4916 21829 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
4917 21829 : unsigned HOST_WIDE_INT nunits;
4918 21829 : unsigned j, number_of_places_left_in_vector;
4919 21829 : unsigned int i;
4920 :
4921 43658 : gcc_assert (group_size == initial_values.length () || neutral_op);
4922 :
4923 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
4924 : created vectors. It is greater than 1 if unrolling is performed.
4925 :
4926 : For example, we have two scalar operands, s1 and s2 (e.g., group of
4927 : strided accesses of size two), while NUNITS is four (i.e., four scalars
4928 : of this type can be packed in a vector). The output vector will contain
4929 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
4930 : will be 2).
4931 :
4932 : If GROUP_SIZE > NUNITS, the scalars will be split into several
4933 : vectors containing the operands.
4934 :
4935 : For example, NUNITS is four as before, and the group size is 8
4936 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
4937 : {s5, s6, s7, s8}. */
4938 :
4939 21829 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
4940 : nunits = group_size;
4941 :
4942 21829 : tree vector_elt_type = TREE_TYPE (vector_type);
4943 21829 : number_of_places_left_in_vector = nunits;
4944 21829 : bool constant_p = true;
4945 21829 : tree_vector_builder elts (vector_type, nunits, 1);
4946 21829 : elts.quick_grow (nunits);
4947 21829 : gimple_seq ctor_seq = NULL;
4948 21829 : if (neutral_op
4949 43066 : && !useless_type_conversion_p (vector_elt_type,
4950 21237 : TREE_TYPE (neutral_op)))
4951 : {
4952 244 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
4953 223 : neutral_op = gimple_build (&ctor_seq, COND_EXPR,
4954 : vector_elt_type,
4955 : neutral_op,
4956 : build_all_ones_cst (vector_elt_type),
4957 : build_zero_cst (vector_elt_type));
4958 : else
4959 21 : neutral_op = gimple_convert (&ctor_seq, vector_elt_type, neutral_op);
4960 : }
4961 204391 : for (j = 0; j < nunits * number_of_vectors; ++j)
4962 : {
4963 182562 : tree op;
4964 182562 : i = j % group_size;
4965 :
4966 : /* Get the def before the loop. In reduction chain we have only
4967 : one initial value. Else we have as many as PHIs in the group. */
4968 182562 : if (i >= initial_values.length () || (j > i && neutral_op))
4969 : op = neutral_op;
4970 : else
4971 : {
4972 51502 : if (!useless_type_conversion_p (vector_elt_type,
4973 25751 : TREE_TYPE (initial_values[i])))
4974 : {
4975 259 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
4976 470 : initial_values[i] = gimple_build (&ctor_seq, COND_EXPR,
4977 : vector_elt_type,
4978 235 : initial_values[i],
4979 : build_all_ones_cst
4980 : (vector_elt_type),
4981 : build_zero_cst
4982 : (vector_elt_type));
4983 : else
4984 48 : initial_values[i] = gimple_convert (&ctor_seq,
4985 : vector_elt_type,
4986 24 : initial_values[i]);
4987 : }
4988 25751 : op = initial_values[i];
4989 : }
4990 :
4991 : /* Create 'vect_ = {op0,op1,...,opn}'. */
4992 182562 : number_of_places_left_in_vector--;
4993 182562 : elts[nunits - number_of_places_left_in_vector - 1] = op;
4994 182562 : if (!CONSTANT_CLASS_P (op))
4995 2487 : constant_p = false;
4996 :
4997 182562 : if (number_of_places_left_in_vector == 0)
4998 : {
4999 23404 : tree init;
5000 46808 : if (constant_p && !neutral_op
5001 46519 : ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type), nunits)
5002 23404 : : known_eq (TYPE_VECTOR_SUBPARTS (vector_type), nunits))
5003 : /* Build the vector directly from ELTS. */
5004 23404 : init = gimple_build_vector (&ctor_seq, &elts);
5005 0 : else if (neutral_op)
5006 : {
5007 : /* Build a vector of the neutral value and shift the
5008 : other elements into place. */
5009 0 : init = gimple_build_vector_from_val (&ctor_seq, vector_type,
5010 : neutral_op);
5011 0 : int k = nunits;
5012 0 : while (k > 0 && operand_equal_p (elts[k - 1], neutral_op))
5013 0 : k -= 1;
5014 0 : while (k > 0)
5015 : {
5016 0 : k -= 1;
5017 0 : init = gimple_build (&ctor_seq, CFN_VEC_SHL_INSERT,
5018 0 : vector_type, init, elts[k]);
5019 : }
5020 : }
5021 : else
5022 : {
5023 : /* First time round, duplicate ELTS to fill the
5024 : required number of vectors. */
5025 0 : duplicate_and_interleave (loop_vinfo, &ctor_seq, vector_type,
5026 : elts, number_of_vectors, *vec_oprnds);
5027 0 : break;
5028 : }
5029 23404 : vec_oprnds->quick_push (init);
5030 :
5031 23404 : number_of_places_left_in_vector = nunits;
5032 23404 : elts.new_vector (vector_type, nunits, 1);
5033 23404 : elts.quick_grow (nunits);
5034 23404 : constant_p = true;
5035 : }
5036 : }
5037 21829 : if (ctor_seq != NULL)
5038 455 : vect_emit_reduction_init_stmts (loop_vinfo, reduc_info, ctor_seq);
5039 21829 : }
5040 :
5041 : vect_reduc_info
5042 218355 : info_for_reduction (loop_vec_info loop_vinfo, slp_tree node)
5043 : {
5044 218355 : if (node->cycle_info.id == -1)
5045 : return NULL;
5046 213467 : return loop_vinfo->reduc_infos[node->cycle_info.id];
5047 : }
5048 :
5049 : /* See if LOOP_VINFO is an epilogue loop whose main loop had a reduction that
5050 : REDUC_INFO can build on. Adjust REDUC_INFO and return true if so, otherwise
5051 : return false. */
5052 :
5053 : static bool
5054 21468 : vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
5055 : vect_reduc_info reduc_info, tree vectype)
5056 : {
5057 21468 : loop_vec_info main_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
5058 21468 : if (!main_loop_vinfo)
5059 : return false;
5060 :
5061 4669 : if (VECT_REDUC_INFO_TYPE (reduc_info) != TREE_CODE_REDUCTION)
5062 : return false;
5063 :
5064 : /* We are not set up to handle vector bools when they are not mapped
5065 : to vector integer data types. */
5066 4654 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5067 4726 : && GET_MODE_CLASS (TYPE_MODE (vectype)) != MODE_VECTOR_INT)
5068 : return false;
5069 :
5070 4652 : unsigned int num_phis = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).length ();
5071 4652 : auto_vec<tree, 16> main_loop_results (num_phis);
5072 4652 : auto_vec<tree, 16> initial_values (num_phis);
5073 4652 : if (edge main_loop_edge = loop_vinfo->main_loop_edge)
5074 : {
5075 : /* The epilogue loop can be entered either from the main loop or
5076 : from an earlier guard block. */
5077 4429 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
5078 17736 : for (tree incoming_value : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info))
5079 : {
5080 : /* Look for:
5081 :
5082 : INCOMING_VALUE = phi<MAIN_LOOP_RESULT(main loop),
5083 : INITIAL_VALUE(guard block)>. */
5084 4449 : gcc_assert (TREE_CODE (incoming_value) == SSA_NAME);
5085 :
5086 4449 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (incoming_value));
5087 4449 : gcc_assert (gimple_bb (phi) == main_loop_edge->dest);
5088 :
5089 4449 : tree from_main_loop = PHI_ARG_DEF_FROM_EDGE (phi, main_loop_edge);
5090 4449 : tree from_skip = PHI_ARG_DEF_FROM_EDGE (phi, skip_edge);
5091 :
5092 4449 : main_loop_results.quick_push (from_main_loop);
5093 4449 : initial_values.quick_push (from_skip);
5094 : }
5095 : }
5096 : else
5097 : /* The main loop dominates the epilogue loop. */
5098 223 : main_loop_results.splice (VECT_REDUC_INFO_INITIAL_VALUES (reduc_info));
5099 :
5100 : /* See if the main loop has the kind of accumulator we need. */
5101 4652 : vect_reusable_accumulator *accumulator
5102 4652 : = main_loop_vinfo->reusable_accumulators.get (main_loop_results[0]);
5103 4652 : if (!accumulator
5104 9288 : || num_phis != VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).length ()
5105 13936 : || !std::equal (main_loop_results.begin (), main_loop_results.end (),
5106 : VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).begin ()))
5107 : return false;
5108 :
5109 : /* Handle the case where we can reduce wider vectors to narrower ones. */
5110 4642 : tree old_vectype = TREE_TYPE (accumulator->reduc_input);
5111 4642 : unsigned HOST_WIDE_INT m;
5112 4642 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (old_vectype),
5113 4642 : TYPE_VECTOR_SUBPARTS (vectype), &m))
5114 0 : return false;
5115 : /* Check the intermediate vector types and operations are available. */
5116 4642 : tree prev_vectype = old_vectype;
5117 4642 : poly_uint64 intermediate_nunits = TYPE_VECTOR_SUBPARTS (old_vectype);
5118 13552 : while (known_gt (intermediate_nunits, TYPE_VECTOR_SUBPARTS (vectype)))
5119 : {
5120 4792 : intermediate_nunits = exact_div (intermediate_nunits, 2);
5121 4792 : tree intermediate_vectype = get_related_vectype_for_scalar_type
5122 4792 : (TYPE_MODE (vectype), TREE_TYPE (vectype), intermediate_nunits);
5123 4792 : if (!intermediate_vectype
5124 4792 : || !directly_supported_p (VECT_REDUC_INFO_CODE (reduc_info),
5125 : intermediate_vectype)
5126 9064 : || !can_vec_extract (TYPE_MODE (prev_vectype),
5127 4272 : TYPE_MODE (intermediate_vectype)))
5128 : return false;
5129 : prev_vectype = intermediate_vectype;
5130 : }
5131 :
5132 : /* Non-SLP reductions might apply an adjustment after the reduction
5133 : operation, in order to simplify the initialization of the accumulator.
5134 : If the epilogue loop carries on from where the main loop left off,
5135 : it should apply the same adjustment to the final reduction result.
5136 :
5137 : If the epilogue loop can also be entered directly (rather than via
5138 : the main loop), we need to be able to handle that case in the same way,
5139 : with the same adjustment. (In principle we could add a PHI node
5140 : to select the correct adjustment, but in practice that shouldn't be
5141 : necessary.) */
5142 4118 : tree main_adjustment
5143 4118 : = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (accumulator->reduc_info);
5144 4118 : if (loop_vinfo->main_loop_edge && main_adjustment)
5145 : {
5146 3432 : gcc_assert (num_phis == 1);
5147 3432 : tree initial_value = initial_values[0];
5148 : /* Check that we can use INITIAL_VALUE as the adjustment and
5149 : initialize the accumulator with a neutral value instead. */
5150 3432 : if (!operand_equal_p (initial_value, main_adjustment))
5151 : return false;
5152 3422 : initial_values[0] = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
5153 : }
5154 4108 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info) = main_adjustment;
5155 4108 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).truncate (0);
5156 4108 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).splice (initial_values);
5157 4108 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info) = accumulator;
5158 4108 : return true;
5159 4652 : }
5160 :
5161 : /* Reduce the vector VEC_DEF down to VECTYPE with reduction operation
5162 : CODE emitting stmts before GSI. Returns a vector def of VECTYPE. */
5163 :
5164 : static tree
5165 4152 : vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
5166 : gimple_seq *seq)
5167 : {
5168 4152 : gcc_assert (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (vec_def))
5169 : || (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (vec_def)))
5170 : == MODE_VECTOR_INT));
5171 4152 : unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def)).to_constant ();
5172 4152 : unsigned nunits1 = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
5173 4152 : tree stype = TREE_TYPE (vectype);
5174 4152 : tree new_temp = vec_def;
5175 8447 : while (nunits > nunits1)
5176 : {
5177 4295 : nunits /= 2;
5178 4295 : tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
5179 4295 : stype, nunits);
5180 4295 : unsigned int bitsize = tree_to_uhwi (TYPE_SIZE (vectype1));
5181 :
5182 : /* The target has to make sure we support lowpart/highpart
5183 : extraction, either via direct vector extract or through
5184 : an integer mode punning. */
5185 4295 : tree dst1, dst2;
5186 4295 : gimple *epilog_stmt;
5187 4295 : if (convert_optab_handler (vec_extract_optab,
5188 4295 : TYPE_MODE (TREE_TYPE (new_temp)),
5189 4295 : TYPE_MODE (vectype1))
5190 : != CODE_FOR_nothing)
5191 : {
5192 : /* Extract sub-vectors directly once vec_extract becomes
5193 : a conversion optab. */
5194 2615 : dst1 = make_ssa_name (vectype1);
5195 2615 : epilog_stmt
5196 5230 : = gimple_build_assign (dst1, BIT_FIELD_REF,
5197 : build3 (BIT_FIELD_REF, vectype1,
5198 2615 : new_temp, TYPE_SIZE (vectype1),
5199 : bitsize_int (0)));
5200 2615 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5201 2615 : dst2 = make_ssa_name (vectype1);
5202 2615 : epilog_stmt
5203 2615 : = gimple_build_assign (dst2, BIT_FIELD_REF,
5204 : build3 (BIT_FIELD_REF, vectype1,
5205 2615 : new_temp, TYPE_SIZE (vectype1),
5206 2615 : bitsize_int (bitsize)));
5207 2615 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5208 : }
5209 : else
5210 : {
5211 : /* Extract via punning to appropriately sized integer mode
5212 : vector. */
5213 1680 : tree eltype = build_nonstandard_integer_type (bitsize, 1);
5214 1680 : tree etype = build_vector_type (eltype, 2);
5215 3360 : gcc_assert (convert_optab_handler (vec_extract_optab,
5216 : TYPE_MODE (etype),
5217 : TYPE_MODE (eltype))
5218 : != CODE_FOR_nothing);
5219 1680 : tree tem = make_ssa_name (etype);
5220 1680 : epilog_stmt = gimple_build_assign (tem, VIEW_CONVERT_EXPR,
5221 : build1 (VIEW_CONVERT_EXPR,
5222 : etype, new_temp));
5223 1680 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5224 1680 : new_temp = tem;
5225 1680 : tem = make_ssa_name (eltype);
5226 1680 : epilog_stmt
5227 3360 : = gimple_build_assign (tem, BIT_FIELD_REF,
5228 : build3 (BIT_FIELD_REF, eltype,
5229 1680 : new_temp, TYPE_SIZE (eltype),
5230 : bitsize_int (0)));
5231 1680 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5232 1680 : dst1 = make_ssa_name (vectype1);
5233 1680 : epilog_stmt = gimple_build_assign (dst1, VIEW_CONVERT_EXPR,
5234 : build1 (VIEW_CONVERT_EXPR,
5235 : vectype1, tem));
5236 1680 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5237 1680 : tem = make_ssa_name (eltype);
5238 1680 : epilog_stmt
5239 1680 : = gimple_build_assign (tem, BIT_FIELD_REF,
5240 : build3 (BIT_FIELD_REF, eltype,
5241 1680 : new_temp, TYPE_SIZE (eltype),
5242 1680 : bitsize_int (bitsize)));
5243 1680 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5244 1680 : dst2 = make_ssa_name (vectype1);
5245 1680 : epilog_stmt = gimple_build_assign (dst2, VIEW_CONVERT_EXPR,
5246 : build1 (VIEW_CONVERT_EXPR,
5247 : vectype1, tem));
5248 1680 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5249 : }
5250 :
5251 4295 : new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
5252 : }
5253 4152 : if (!useless_type_conversion_p (vectype, TREE_TYPE (new_temp)))
5254 : {
5255 66 : tree dst3 = make_ssa_name (vectype);
5256 66 : gimple *epilog_stmt = gimple_build_assign (dst3, VIEW_CONVERT_EXPR,
5257 : build1 (VIEW_CONVERT_EXPR,
5258 : vectype, new_temp));
5259 66 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5260 66 : new_temp = dst3;
5261 : }
5262 :
5263 4152 : return new_temp;
5264 : }
5265 :
5266 : /* Function vect_create_epilog_for_reduction
5267 :
5268 : Create code at the loop-epilog to finalize the result of a reduction
5269 : computation.
5270 :
5271 : STMT_INFO is the scalar reduction stmt that is being vectorized.
5272 : SLP_NODE is an SLP node containing a group of reduction statements. The
5273 : first one in this group is STMT_INFO.
5274 : SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
5275 : REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
5276 : (counting from 0)
5277 : LOOP_EXIT is the edge to update in the merge block. In the case of a single
5278 : exit this edge is always the main loop exit.
5279 :
5280 : This function:
5281 : 1. Completes the reduction def-use cycles.
5282 : 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
5283 : by calling the function specified by REDUC_FN if available, or by
5284 : other means (whole-vector shifts or a scalar loop).
5285 : The function also creates a new phi node at the loop exit to preserve
5286 : loop-closed form, as illustrated below.
5287 :
5288 : The flow at the entry to this function:
5289 :
5290 : loop:
5291 : vec_def = phi <vec_init, null> # REDUCTION_PHI
5292 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5293 : s_loop = scalar_stmt # (scalar) STMT_INFO
5294 : loop_exit:
5295 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5296 : use <s_out0>
5297 : use <s_out0>
5298 :
5299 : The above is transformed by this function into:
5300 :
5301 : loop:
5302 : vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
5303 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5304 : s_loop = scalar_stmt # (scalar) STMT_INFO
5305 : loop_exit:
5306 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5307 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5308 : v_out2 = reduce <v_out1>
5309 : s_out3 = extract_field <v_out2, 0>
5310 : s_out4 = adjust_result <s_out3>
5311 : use <s_out4>
5312 : use <s_out4>
5313 : */
5314 :
5315 : static void
5316 22176 : vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
5317 : stmt_vec_info stmt_info,
5318 : slp_tree slp_node,
5319 : slp_instance slp_node_instance,
5320 : edge loop_exit)
5321 : {
5322 22176 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
5323 22176 : code_helper code = VECT_REDUC_INFO_CODE (reduc_info);
5324 22176 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
5325 22176 : tree vectype;
5326 22176 : machine_mode mode;
5327 22176 : basic_block exit_bb;
5328 22176 : gimple *new_phi = NULL, *phi = NULL;
5329 22176 : gimple_stmt_iterator exit_gsi;
5330 22176 : tree new_temp = NULL_TREE, new_name, new_scalar_dest;
5331 22176 : gimple *epilog_stmt = NULL;
5332 22176 : gimple *exit_phi;
5333 22176 : tree def;
5334 22176 : tree orig_name, scalar_result;
5335 22176 : imm_use_iterator imm_iter;
5336 22176 : use_operand_p use_p;
5337 22176 : gimple *use_stmt;
5338 22176 : auto_vec<tree> reduc_inputs;
5339 22176 : int j, i;
5340 22176 : vec<tree> &scalar_results = VECT_REDUC_INFO_SCALAR_RESULTS (reduc_info);
5341 22176 : unsigned int k;
5342 : /* SLP reduction without reduction chain, e.g.,
5343 : # a1 = phi <a2, a0>
5344 : # b1 = phi <b2, b0>
5345 : a2 = operation (a1)
5346 : b2 = operation (b1) */
5347 22176 : const bool slp_reduc = !reduc_info->is_reduc_chain;
5348 22176 : tree induction_index = NULL_TREE;
5349 :
5350 22176 : unsigned int group_size = SLP_TREE_LANES (slp_node);
5351 :
5352 22176 : bool double_reduc = false;
5353 22176 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
5354 22176 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
5355 : {
5356 0 : double_reduc = true;
5357 0 : gcc_assert (slp_reduc);
5358 : }
5359 :
5360 22176 : vectype = VECT_REDUC_INFO_VECTYPE (reduc_info);
5361 22176 : gcc_assert (vectype);
5362 22176 : mode = TYPE_MODE (vectype);
5363 :
5364 22176 : tree induc_val = NULL_TREE;
5365 22176 : tree adjustment_def = NULL;
5366 : /* Optimize: for induction condition reduction, if we can't use zero
5367 : for induc_val, use initial_def. */
5368 22176 : if (VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5369 62 : induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
5370 22114 : else if (double_reduc)
5371 : ;
5372 : else
5373 22114 : adjustment_def = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info);
5374 :
5375 22176 : stmt_vec_info single_live_out_stmt[] = { stmt_info };
5376 22176 : array_slice<const stmt_vec_info> live_out_stmts = single_live_out_stmt;
5377 22176 : if (slp_reduc)
5378 : /* All statements produce live-out values. */
5379 43914 : live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
5380 :
5381 22176 : unsigned vec_num
5382 22176 : = SLP_TREE_VEC_DEFS (slp_node_instance->reduc_phis).length ();
5383 :
5384 : /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
5385 : which is updated with the current index of the loop for every match of
5386 : the original loop's cond_expr (VEC_STMT). This results in a vector
5387 : containing the last time the condition passed for that vector lane.
5388 : The first match will be a 1 to allow 0 to be used for non-matching
5389 : indexes. If there are no matches at all then the vector will be all
5390 : zeroes.
5391 :
5392 : PR92772: This algorithm is broken for architectures that support
5393 : masked vectors, but do not provide fold_extract_last. */
5394 22176 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION)
5395 : {
5396 87 : gcc_assert (!double_reduc);
5397 87 : auto_vec<std::pair<tree, bool>, 2> ccompares;
5398 87 : slp_tree cond_node = slp_node_instance->root;
5399 183 : while (cond_node != slp_node_instance->reduc_phis)
5400 : {
5401 96 : stmt_vec_info cond_info = SLP_TREE_REPRESENTATIVE (cond_node);
5402 96 : if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
5403 : {
5404 96 : gimple *vec_stmt
5405 96 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (cond_node)[0]);
5406 96 : gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
5407 96 : ccompares.safe_push
5408 96 : (std::make_pair (gimple_assign_rhs1 (vec_stmt),
5409 96 : SLP_TREE_REDUC_IDX (cond_node) == 2));
5410 : }
5411 96 : int slp_reduc_idx = SLP_TREE_REDUC_IDX (cond_node);
5412 96 : cond_node = SLP_TREE_CHILDREN (cond_node)[slp_reduc_idx];
5413 : }
5414 87 : gcc_assert (ccompares.length () != 0);
5415 :
5416 87 : tree indx_before_incr, indx_after_incr;
5417 87 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
5418 87 : int scalar_precision
5419 87 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype)));
5420 87 : tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
5421 87 : tree cr_index_vector_type = get_related_vectype_for_scalar_type
5422 87 : (TYPE_MODE (vectype), cr_index_scalar_type,
5423 : TYPE_VECTOR_SUBPARTS (vectype));
5424 :
5425 : /* First we create a simple vector induction variable which starts
5426 : with the values {1,2,3,...} (SERIES_VECT) and increments by the
5427 : vector size (STEP). */
5428 :
5429 : /* Create a {1,2,3,...} vector. */
5430 87 : tree series_vect = build_index_vector (cr_index_vector_type, 1, 1);
5431 :
5432 : /* Create a vector of the step value. */
5433 87 : tree step = build_int_cst (cr_index_scalar_type, nunits_out);
5434 87 : tree vec_step = build_vector_from_val (cr_index_vector_type, step);
5435 :
5436 : /* Create an induction variable. */
5437 87 : gimple_stmt_iterator incr_gsi;
5438 87 : bool insert_after;
5439 87 : vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
5440 : &incr_gsi, &insert_after);
5441 87 : create_iv (series_vect, PLUS_EXPR, vec_step, NULL_TREE, loop, &incr_gsi,
5442 : insert_after, &indx_before_incr, &indx_after_incr);
5443 :
5444 : /* Next create a new phi node vector (NEW_PHI_TREE) which starts
5445 : filled with zeros (VEC_ZERO). */
5446 :
5447 : /* Create a vector of 0s. */
5448 87 : tree zero = build_zero_cst (cr_index_scalar_type);
5449 87 : tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
5450 :
5451 : /* Create a vector phi node. */
5452 87 : tree new_phi_tree = make_ssa_name (cr_index_vector_type);
5453 87 : new_phi = create_phi_node (new_phi_tree, loop->header);
5454 87 : add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
5455 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
5456 :
5457 : /* Now take the condition from the loops original cond_exprs
5458 : and produce a new cond_exprs (INDEX_COND_EXPR) which for
5459 : every match uses values from the induction variable
5460 : (INDEX_BEFORE_INCR) otherwise uses values from the phi node
5461 : (NEW_PHI_TREE).
5462 : Finally, we update the phi (NEW_PHI_TREE) to take the value of
5463 : the new cond_expr (INDEX_COND_EXPR). */
5464 87 : gimple_seq stmts = NULL;
5465 270 : for (int i = ccompares.length () - 1; i != -1; --i)
5466 : {
5467 96 : tree ccompare = ccompares[i].first;
5468 96 : if (ccompares[i].second)
5469 69 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5470 : cr_index_vector_type,
5471 : ccompare,
5472 : indx_before_incr, new_phi_tree);
5473 : else
5474 27 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5475 : cr_index_vector_type,
5476 : ccompare,
5477 : new_phi_tree, indx_before_incr);
5478 : }
5479 87 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
5480 :
5481 : /* Update the phi with the vec cond. */
5482 87 : induction_index = new_phi_tree;
5483 87 : add_phi_arg (as_a <gphi *> (new_phi), induction_index,
5484 : loop_latch_edge (loop), UNKNOWN_LOCATION);
5485 87 : }
5486 :
5487 : /* 2. Create epilog code.
5488 : The reduction epilog code operates across the elements of the vector
5489 : of partial results computed by the vectorized loop.
5490 : The reduction epilog code consists of:
5491 :
5492 : step 1: compute the scalar result in a vector (v_out2)
5493 : step 2: extract the scalar result (s_out3) from the vector (v_out2)
5494 : step 3: adjust the scalar result (s_out3) if needed.
5495 :
5496 : Step 1 can be accomplished using one the following three schemes:
5497 : (scheme 1) using reduc_fn, if available.
5498 : (scheme 2) using whole-vector shifts, if available.
5499 : (scheme 3) using a scalar loop. In this case steps 1+2 above are
5500 : combined.
5501 :
5502 : The overall epilog code looks like this:
5503 :
5504 : s_out0 = phi <s_loop> # original EXIT_PHI
5505 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5506 : v_out2 = reduce <v_out1> # step 1
5507 : s_out3 = extract_field <v_out2, 0> # step 2
5508 : s_out4 = adjust_result <s_out3> # step 3
5509 :
5510 : (step 3 is optional, and steps 1 and 2 may be combined).
5511 : Lastly, the uses of s_out0 are replaced by s_out4. */
5512 :
5513 :
5514 : /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
5515 : v_out1 = phi <VECT_DEF>
5516 : Store them in NEW_PHIS. */
5517 : /* We need to reduce values in all exits. */
5518 22176 : exit_bb = loop_exit->dest;
5519 22176 : exit_gsi = gsi_after_labels (exit_bb);
5520 22176 : reduc_inputs.create (vec_num);
5521 68113 : for (unsigned i = 0; i < vec_num; i++)
5522 : {
5523 23761 : gimple_seq stmts = NULL;
5524 23761 : def = vect_get_slp_vect_def (slp_node, i);
5525 23761 : tree new_def = copy_ssa_name (def);
5526 23761 : phi = create_phi_node (new_def, exit_bb);
5527 23761 : if (LOOP_VINFO_MAIN_EXIT (loop_vinfo) == loop_exit)
5528 23734 : SET_PHI_ARG_DEF (phi, loop_exit->dest_idx, def);
5529 : else
5530 : {
5531 57 : for (unsigned k = 0; k < gimple_phi_num_args (phi); k++)
5532 30 : SET_PHI_ARG_DEF (phi, k, def);
5533 : }
5534 23761 : new_def = gimple_convert (&stmts, vectype, new_def);
5535 23761 : reduc_inputs.quick_push (new_def);
5536 23761 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5537 : }
5538 :
5539 : /* 2.2 Get the original scalar reduction variable as defined in the loop.
5540 : In case STMT is a "pattern-stmt" (i.e. - it represents a reduction
5541 : pattern), the scalar-def is taken from the original stmt that the
5542 : pattern-stmt (STMT) replaces. */
5543 :
5544 23004 : tree scalar_dest = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5545 22176 : tree scalar_type = TREE_TYPE (scalar_dest);
5546 22176 : scalar_results.truncate (0);
5547 22176 : scalar_results.reserve_exact (group_size);
5548 22176 : new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
5549 :
5550 : /* True if we should implement SLP_REDUC using native reduction operations
5551 : instead of scalar operations. */
5552 22176 : const bool direct_slp_reduc
5553 22176 : = (reduc_fn != IFN_LAST
5554 22176 : && slp_reduc
5555 22176 : && !TYPE_VECTOR_SUBPARTS (vectype).is_constant ());
5556 :
5557 : /* If signed overflow is undefined we might need to perform reduction
5558 : computations in an unsigned type. */
5559 22176 : tree compute_vectype = vectype;
5560 22176 : if (ANY_INTEGRAL_TYPE_P (vectype)
5561 15086 : && TYPE_OVERFLOW_UNDEFINED (vectype)
5562 5626 : && code.is_tree_code ()
5563 27802 : && arith_code_with_undefined_signed_overflow ((tree_code) code))
5564 4134 : compute_vectype = unsigned_type_for (vectype);
5565 :
5566 : /* In case of reduction chain, e.g.,
5567 : # a1 = phi <a3, a0>
5568 : a2 = operation (a1)
5569 : a3 = operation (a2),
5570 :
5571 : we may end up with more than one vector result. Here we reduce them
5572 : to one vector.
5573 :
5574 : The same is true for a SLP reduction, e.g.,
5575 : # a1 = phi <a2, a0>
5576 : # b1 = phi <b2, b0>
5577 : a2 = operation (a1)
5578 : b2 = operation (a2),
5579 :
5580 : where we can end up with more than one vector as well. We can
5581 : easily accumulate vectors when the number of vector elements is
5582 : a multiple of the SLP group size.
5583 :
5584 : The same is true if we couldn't use a single defuse cycle. */
5585 22176 : if ((!slp_reduc
5586 : || direct_slp_reduc
5587 : || (slp_reduc
5588 22176 : && constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype), group_size)))
5589 44352 : && reduc_inputs.length () > 1)
5590 : {
5591 543 : gimple_seq stmts = NULL;
5592 543 : tree single_input = reduc_inputs[0];
5593 543 : if (compute_vectype != vectype)
5594 157 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5595 : compute_vectype, single_input);
5596 1973 : for (k = 1; k < reduc_inputs.length (); k++)
5597 : {
5598 1430 : tree input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5599 1430 : compute_vectype, reduc_inputs[k]);
5600 1430 : single_input = gimple_build (&stmts, code, compute_vectype,
5601 : single_input, input);
5602 : }
5603 543 : if (compute_vectype != vectype)
5604 157 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5605 : vectype, single_input);
5606 543 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5607 :
5608 543 : reduc_inputs.truncate (0);
5609 543 : reduc_inputs.safe_push (single_input);
5610 : }
5611 :
5612 22176 : tree orig_reduc_input = reduc_inputs[0];
5613 :
5614 : /* If this loop is an epilogue loop that can be skipped after the
5615 : main loop, we can only share a reduction operation between the
5616 : main loop and the epilogue if we put it at the target of the
5617 : skip edge.
5618 :
5619 : We can still reuse accumulators if this check fails. Doing so has
5620 : the minor(?) benefit of making the epilogue loop's scalar result
5621 : independent of the main loop's scalar result. */
5622 22176 : bool unify_with_main_loop_p = false;
5623 22176 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
5624 4108 : && loop_vinfo->skip_this_loop_edge
5625 3868 : && single_succ_p (exit_bb)
5626 22193 : && single_succ (exit_bb) == loop_vinfo->skip_this_loop_edge->dest)
5627 : {
5628 17 : unify_with_main_loop_p = true;
5629 :
5630 17 : basic_block reduc_block = loop_vinfo->skip_this_loop_edge->dest;
5631 17 : reduc_inputs[0] = make_ssa_name (vectype);
5632 17 : gphi *new_phi = create_phi_node (reduc_inputs[0], reduc_block);
5633 17 : add_phi_arg (new_phi, orig_reduc_input, single_succ_edge (exit_bb),
5634 : UNKNOWN_LOCATION);
5635 17 : add_phi_arg (new_phi,
5636 17 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)->reduc_input,
5637 : loop_vinfo->skip_this_loop_edge, UNKNOWN_LOCATION);
5638 17 : exit_gsi = gsi_after_labels (reduc_block);
5639 : }
5640 :
5641 : /* Shouldn't be used beyond this point. */
5642 22176 : exit_bb = nullptr;
5643 :
5644 : /* If we are operating on a mask vector and do not support direct mask
5645 : reduction, work on a bool data vector instead of a mask vector. */
5646 22176 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5647 251 : && VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info)
5648 22377 : && vectype != VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info))
5649 : {
5650 201 : compute_vectype = vectype = VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info);
5651 201 : gimple_seq stmts = NULL;
5652 410 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
5653 418 : reduc_inputs[i] = gimple_build (&stmts, VEC_COND_EXPR, vectype,
5654 209 : reduc_inputs[i],
5655 : build_one_cst (vectype),
5656 : build_zero_cst (vectype));
5657 201 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5658 : }
5659 :
5660 22176 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5661 87 : && reduc_fn != IFN_LAST)
5662 : {
5663 : /* For condition reductions, we have a vector (REDUC_INPUTS 0) containing
5664 : various data values where the condition matched and another vector
5665 : (INDUCTION_INDEX) containing all the indexes of those matches. We
5666 : need to extract the last matching index (which will be the index with
5667 : highest value) and use this to index into the data vector.
5668 : For the case where there were no matches, the data vector will contain
5669 : all default values and the index vector will be all zeros. */
5670 :
5671 : /* Get various versions of the type of the vector of indexes. */
5672 14 : tree index_vec_type = TREE_TYPE (induction_index);
5673 14 : gcc_checking_assert (TYPE_UNSIGNED (index_vec_type));
5674 14 : tree index_scalar_type = TREE_TYPE (index_vec_type);
5675 14 : tree index_vec_cmp_type = truth_type_for (index_vec_type);
5676 :
5677 : /* Get an unsigned integer version of the type of the data vector. */
5678 14 : int scalar_precision
5679 14 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
5680 14 : tree scalar_type_unsigned = make_unsigned_type (scalar_precision);
5681 14 : tree vectype_unsigned = get_same_sized_vectype (scalar_type_unsigned,
5682 : vectype);
5683 :
5684 : /* First we need to create a vector (ZERO_VEC) of zeros and another
5685 : vector (MAX_INDEX_VEC) filled with the last matching index, which we
5686 : can create using a MAX reduction and then expanding.
5687 : In the case where the loop never made any matches, the max index will
5688 : be zero. */
5689 :
5690 : /* Vector of {0, 0, 0,...}. */
5691 14 : tree zero_vec = build_zero_cst (vectype);
5692 :
5693 : /* Find maximum value from the vector of found indexes. */
5694 14 : tree max_index = make_ssa_name (index_scalar_type);
5695 14 : gcall *max_index_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5696 : 1, induction_index);
5697 14 : gimple_call_set_lhs (max_index_stmt, max_index);
5698 14 : gsi_insert_before (&exit_gsi, max_index_stmt, GSI_SAME_STMT);
5699 :
5700 : /* Vector of {max_index, max_index, max_index,...}. */
5701 14 : tree max_index_vec = make_ssa_name (index_vec_type);
5702 14 : tree max_index_vec_rhs = build_vector_from_val (index_vec_type,
5703 : max_index);
5704 14 : gimple *max_index_vec_stmt = gimple_build_assign (max_index_vec,
5705 : max_index_vec_rhs);
5706 14 : gsi_insert_before (&exit_gsi, max_index_vec_stmt, GSI_SAME_STMT);
5707 :
5708 : /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
5709 : with the vector (INDUCTION_INDEX) of found indexes, choosing values
5710 : from the data vector (REDUC_INPUTS 0) for matches, 0 (ZERO_VEC)
5711 : otherwise. Only one value should match, resulting in a vector
5712 : (VEC_COND) with one data value and the rest zeros.
5713 : In the case where the loop never made any matches, every index will
5714 : match, resulting in a vector with all data values (which will all be
5715 : the default value). */
5716 :
5717 : /* Compare the max index vector to the vector of found indexes to find
5718 : the position of the max value. */
5719 14 : tree vec_compare = make_ssa_name (index_vec_cmp_type);
5720 14 : gimple *vec_compare_stmt = gimple_build_assign (vec_compare, EQ_EXPR,
5721 : induction_index,
5722 : max_index_vec);
5723 14 : gsi_insert_before (&exit_gsi, vec_compare_stmt, GSI_SAME_STMT);
5724 :
5725 : /* Use the compare to choose either values from the data vector or
5726 : zero. */
5727 14 : tree vec_cond = make_ssa_name (vectype);
5728 14 : gimple *vec_cond_stmt = gimple_build_assign (vec_cond, VEC_COND_EXPR,
5729 : vec_compare,
5730 14 : reduc_inputs[0],
5731 : zero_vec);
5732 14 : gsi_insert_before (&exit_gsi, vec_cond_stmt, GSI_SAME_STMT);
5733 :
5734 : /* Finally we need to extract the data value from the vector (VEC_COND)
5735 : into a scalar (MATCHED_DATA_REDUC). Logically we want to do a OR
5736 : reduction, but because this doesn't exist, we can use a MAX reduction
5737 : instead. The data value might be signed or a float so we need to cast
5738 : it first.
5739 : In the case where the loop never made any matches, the data values are
5740 : all identical, and so will reduce down correctly. */
5741 :
5742 : /* Make the matched data values unsigned. */
5743 14 : tree vec_cond_cast = make_ssa_name (vectype_unsigned);
5744 14 : tree vec_cond_cast_rhs = build1 (VIEW_CONVERT_EXPR, vectype_unsigned,
5745 : vec_cond);
5746 14 : gimple *vec_cond_cast_stmt = gimple_build_assign (vec_cond_cast,
5747 : VIEW_CONVERT_EXPR,
5748 : vec_cond_cast_rhs);
5749 14 : gsi_insert_before (&exit_gsi, vec_cond_cast_stmt, GSI_SAME_STMT);
5750 :
5751 : /* Reduce down to a scalar value. */
5752 14 : tree data_reduc = make_ssa_name (scalar_type_unsigned);
5753 14 : gcall *data_reduc_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5754 : 1, vec_cond_cast);
5755 14 : gimple_call_set_lhs (data_reduc_stmt, data_reduc);
5756 14 : gsi_insert_before (&exit_gsi, data_reduc_stmt, GSI_SAME_STMT);
5757 :
5758 : /* Convert the reduced value back to the result type and set as the
5759 : result. */
5760 14 : gimple_seq stmts = NULL;
5761 14 : new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR, scalar_type,
5762 : data_reduc);
5763 14 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5764 14 : scalar_results.safe_push (new_temp);
5765 14 : }
5766 22162 : else if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5767 73 : && reduc_fn == IFN_LAST)
5768 : {
5769 : /* Condition reduction without supported IFN_REDUC_MAX. Generate
5770 : idx = 0;
5771 : idx_val = induction_index[0];
5772 : val = data_reduc[0];
5773 : for (idx = 0, val = init, i = 0; i < nelts; ++i)
5774 : if (induction_index[i] > idx_val)
5775 : val = data_reduc[i], idx_val = induction_index[i];
5776 : return val; */
5777 :
5778 73 : tree data_eltype = TREE_TYPE (vectype);
5779 73 : tree idx_eltype = TREE_TYPE (TREE_TYPE (induction_index));
5780 73 : unsigned HOST_WIDE_INT el_size = tree_to_uhwi (TYPE_SIZE (idx_eltype));
5781 73 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index));
5782 : /* Enforced by vectorizable_reduction, which ensures we have target
5783 : support before allowing a conditional reduction on variable-length
5784 : vectors. */
5785 73 : unsigned HOST_WIDE_INT v_size = el_size * nunits.to_constant ();
5786 73 : tree idx_val = NULL_TREE, val = NULL_TREE;
5787 469 : for (unsigned HOST_WIDE_INT off = 0; off < v_size; off += el_size)
5788 : {
5789 396 : tree old_idx_val = idx_val;
5790 396 : tree old_val = val;
5791 396 : idx_val = make_ssa_name (idx_eltype);
5792 396 : epilog_stmt = gimple_build_assign (idx_val, BIT_FIELD_REF,
5793 : build3 (BIT_FIELD_REF, idx_eltype,
5794 : induction_index,
5795 396 : bitsize_int (el_size),
5796 396 : bitsize_int (off)));
5797 396 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5798 396 : val = make_ssa_name (data_eltype);
5799 792 : epilog_stmt = gimple_build_assign (val, BIT_FIELD_REF,
5800 : build3 (BIT_FIELD_REF,
5801 : data_eltype,
5802 396 : reduc_inputs[0],
5803 396 : bitsize_int (el_size),
5804 396 : bitsize_int (off)));
5805 396 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5806 396 : if (off != 0)
5807 : {
5808 323 : tree new_idx_val = idx_val;
5809 323 : if (off != v_size - el_size)
5810 : {
5811 250 : new_idx_val = make_ssa_name (idx_eltype);
5812 250 : epilog_stmt = gimple_build_assign (new_idx_val,
5813 : MAX_EXPR, idx_val,
5814 : old_idx_val);
5815 250 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5816 : }
5817 323 : tree cond = make_ssa_name (boolean_type_node);
5818 323 : epilog_stmt = gimple_build_assign (cond, GT_EXPR,
5819 : idx_val, old_idx_val);
5820 323 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5821 323 : tree new_val = make_ssa_name (data_eltype);
5822 323 : epilog_stmt = gimple_build_assign (new_val, COND_EXPR,
5823 : cond, val, old_val);
5824 323 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5825 323 : idx_val = new_idx_val;
5826 323 : val = new_val;
5827 : }
5828 : }
5829 : /* Convert the reduced value back to the result type and set as the
5830 : result. */
5831 73 : gimple_seq stmts = NULL;
5832 73 : val = gimple_convert (&stmts, scalar_type, val);
5833 73 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5834 73 : scalar_results.safe_push (val);
5835 73 : }
5836 :
5837 : /* 2.3 Create the reduction code, using one of the three schemes described
5838 : above. In SLP we simply need to extract all the elements from the
5839 : vector (without reducing them), so we use scalar shifts. */
5840 22089 : else if (reduc_fn != IFN_LAST && (!slp_reduc || group_size == 1))
5841 : {
5842 20125 : tree tmp;
5843 20125 : tree vec_elem_type;
5844 :
5845 : /* Case 1: Create:
5846 : v_out2 = reduc_expr <v_out1> */
5847 :
5848 20125 : if (dump_enabled_p ())
5849 1513 : dump_printf_loc (MSG_NOTE, vect_location,
5850 : "Reduce using direct vector reduction.\n");
5851 :
5852 20125 : gimple_seq stmts = NULL;
5853 20125 : vec_elem_type = TREE_TYPE (vectype);
5854 20125 : new_temp = gimple_build (&stmts, as_combined_fn (reduc_fn),
5855 20125 : vec_elem_type, reduc_inputs[0]);
5856 20125 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
5857 20125 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5858 :
5859 20125 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5860 62 : && induc_val)
5861 : {
5862 : /* Earlier we set the initial value to be a vector if induc_val
5863 : values. Check the result and if it is induc_val then replace
5864 : with the original initial value, unless induc_val is
5865 : the same as initial_def already. */
5866 60 : tree zcompare = make_ssa_name (boolean_type_node);
5867 60 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
5868 : new_temp, induc_val);
5869 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5870 60 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
5871 60 : tmp = make_ssa_name (new_scalar_dest);
5872 60 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
5873 : initial_def, new_temp);
5874 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5875 60 : new_temp = tmp;
5876 : }
5877 :
5878 20125 : scalar_results.safe_push (new_temp);
5879 20125 : }
5880 1777 : else if (direct_slp_reduc)
5881 : {
5882 : /* Here we create one vector for each of the GROUP_SIZE results,
5883 : with the elements for other SLP statements replaced with the
5884 : neutral value. We can then do a normal reduction on each vector. */
5885 :
5886 : /* Enforced by vectorizable_reduction. */
5887 : gcc_assert (reduc_inputs.length () == 1);
5888 : gcc_assert (pow2p_hwi (group_size));
5889 :
5890 : gimple_seq seq = NULL;
5891 :
5892 : /* Build a vector {0, 1, 2, ...}, with the same number of elements
5893 : and the same element size as VECTYPE. */
5894 : tree index = build_index_vector (vectype, 0, 1);
5895 : tree index_type = TREE_TYPE (index);
5896 : tree index_elt_type = TREE_TYPE (index_type);
5897 : tree mask_type = truth_type_for (index_type);
5898 :
5899 : /* Create a vector that, for each element, identifies which of
5900 : the results should use it. */
5901 : tree index_mask = build_int_cst (index_elt_type, group_size - 1);
5902 : index = gimple_build (&seq, BIT_AND_EXPR, index_type, index,
5903 : build_vector_from_val (index_type, index_mask));
5904 :
5905 : /* Get a neutral vector value. This is simply a splat of the neutral
5906 : scalar value if we have one, otherwise the initial scalar value
5907 : is itself a neutral value. */
5908 : tree vector_identity = NULL_TREE;
5909 : tree neutral_op = neutral_op_for_reduction (TREE_TYPE (vectype), code,
5910 : NULL_TREE, false);
5911 : if (neutral_op)
5912 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
5913 : neutral_op);
5914 : for (unsigned int i = 0; i < group_size; ++i)
5915 : {
5916 : /* If there's no universal neutral value, we can use the
5917 : initial scalar value from the original PHI. This is used
5918 : for MIN and MAX reduction, for example. */
5919 : if (!neutral_op)
5920 : {
5921 : tree scalar_value
5922 : = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[i];
5923 : scalar_value = gimple_convert (&seq, TREE_TYPE (vectype),
5924 : scalar_value);
5925 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
5926 : scalar_value);
5927 : }
5928 :
5929 : /* Calculate the equivalent of:
5930 :
5931 : sel[j] = (index[j] == i);
5932 :
5933 : which selects the elements of REDUC_INPUTS[0] that should
5934 : be included in the result. */
5935 : tree compare_val = build_int_cst (index_elt_type, i);
5936 : compare_val = build_vector_from_val (index_type, compare_val);
5937 : tree sel = gimple_build (&seq, EQ_EXPR, mask_type,
5938 : index, compare_val);
5939 :
5940 : /* Calculate the equivalent of:
5941 :
5942 : vec = seq ? reduc_inputs[0] : vector_identity;
5943 :
5944 : VEC is now suitable for a full vector reduction. */
5945 : tree vec = gimple_build (&seq, VEC_COND_EXPR, vectype,
5946 : sel, reduc_inputs[0], vector_identity);
5947 :
5948 : /* Do the reduction and convert it to the appropriate type. */
5949 : tree scalar = gimple_build (&seq, as_combined_fn (reduc_fn),
5950 : TREE_TYPE (vectype), vec);
5951 : scalar = gimple_convert (&seq, scalar_type, scalar);
5952 : scalar_results.safe_push (scalar);
5953 : }
5954 : gsi_insert_seq_before (&exit_gsi, seq, GSI_SAME_STMT);
5955 : }
5956 : else
5957 : {
5958 1777 : bool reduce_with_shift;
5959 1777 : tree vec_temp;
5960 :
5961 1777 : gcc_assert (slp_reduc || reduc_inputs.length () == 1);
5962 :
5963 : /* See if the target wants to do the final (shift) reduction
5964 : in a vector mode of smaller size and first reduce upper/lower
5965 : halves against each other. */
5966 1964 : enum machine_mode mode1 = mode;
5967 1964 : tree stype = TREE_TYPE (vectype);
5968 1964 : if (compute_vectype != vectype)
5969 : {
5970 544 : stype = unsigned_type_for (stype);
5971 544 : gimple_seq stmts = NULL;
5972 1146 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
5973 : {
5974 602 : tree new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5975 602 : compute_vectype, reduc_inputs[i]);
5976 602 : reduc_inputs[i] = new_temp;
5977 : }
5978 544 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5979 : }
5980 1964 : unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
5981 1964 : unsigned nunits1 = nunits;
5982 1964 : if ((mode1 = targetm.vectorize.split_reduction (mode)) != mode
5983 1964 : && reduc_inputs.length () == 1)
5984 : {
5985 41 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
5986 : /* For SLP reductions we have to make sure lanes match up, but
5987 : since we're doing individual element final reduction reducing
5988 : vector width here is even more important.
5989 : ??? We can also separate lanes with permutes, for the common
5990 : case of power-of-two group-size odd/even extracts would work. */
5991 41 : if (slp_reduc && nunits != nunits1)
5992 : {
5993 41 : nunits1 = least_common_multiple (nunits1, group_size);
5994 82 : gcc_assert (exact_log2 (nunits1) != -1 && nunits1 <= nunits);
5995 : }
5996 : }
5997 1923 : else if (!slp_reduc
5998 1923 : && (mode1 = targetm.vectorize.split_reduction (mode)) != mode)
5999 0 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
6000 :
6001 1964 : tree vectype1 = compute_vectype;
6002 1964 : if (mode1 != mode)
6003 : {
6004 47 : vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
6005 47 : stype, nunits1);
6006 : /* First reduce the vector to the desired vector size we should
6007 : do shift reduction on by combining upper and lower halves. */
6008 47 : gimple_seq stmts = NULL;
6009 47 : new_temp = vect_create_partial_epilog (reduc_inputs[0], vectype1,
6010 : code, &stmts);
6011 47 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6012 47 : reduc_inputs[0] = new_temp;
6013 : }
6014 :
6015 1964 : reduce_with_shift = have_whole_vector_shift (mode1);
6016 733 : if (!VECTOR_MODE_P (mode1)
6017 2695 : || !directly_supported_p (code, vectype1))
6018 : reduce_with_shift = false;
6019 :
6020 1947 : if (reduce_with_shift && (!slp_reduc || group_size == 1))
6021 : {
6022 1727 : int element_bitsize = vector_element_bits (vectype1);
6023 : /* Enforced by vectorizable_reduction, which disallows SLP reductions
6024 : for variable-length vectors and also requires direct target support
6025 : for loop reductions. */
6026 1727 : int nelements = TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6027 1727 : vec_perm_builder sel;
6028 1727 : vec_perm_indices indices;
6029 :
6030 1727 : int elt_offset;
6031 :
6032 1727 : tree zero_vec = build_zero_cst (vectype1);
6033 : /* Case 2: Create:
6034 : for (offset = nelements/2; offset >= 1; offset/=2)
6035 : {
6036 : Create: va' = vec_shift <va, offset>
6037 : Create: va = vop <va, va'>
6038 : } */
6039 :
6040 1727 : if (dump_enabled_p ())
6041 368 : dump_printf_loc (MSG_NOTE, vect_location,
6042 : "Reduce using vector shifts\n");
6043 :
6044 1727 : gimple_seq stmts = NULL;
6045 1727 : new_temp = gimple_convert (&stmts, vectype1, reduc_inputs[0]);
6046 1727 : for (elt_offset = nelements / 2;
6047 3760 : elt_offset >= 1;
6048 2033 : elt_offset /= 2)
6049 : {
6050 2033 : calc_vec_perm_mask_for_shift (elt_offset, nelements, &sel);
6051 2033 : indices.new_vector (sel, 2, nelements);
6052 2033 : tree mask = vect_gen_perm_mask_any (vectype1, indices);
6053 2033 : new_name = gimple_build (&stmts, VEC_PERM_EXPR, vectype1,
6054 : new_temp, zero_vec, mask);
6055 2033 : new_temp = gimple_build (&stmts, code,
6056 : vectype1, new_name, new_temp);
6057 : }
6058 :
6059 : /* 2.4 Extract the final scalar result. Create:
6060 : s_out3 = extract_field <v_out2, bitpos> */
6061 :
6062 1727 : if (dump_enabled_p ())
6063 368 : dump_printf_loc (MSG_NOTE, vect_location,
6064 : "extract scalar result\n");
6065 :
6066 1727 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype1),
6067 1727 : new_temp, bitsize_int (element_bitsize),
6068 1727 : bitsize_zero_node);
6069 1727 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6070 1727 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6071 1727 : scalar_results.safe_push (new_temp);
6072 1727 : }
6073 : else
6074 : {
6075 : /* Case 3: Create:
6076 : s = extract_field <v_out2, 0>
6077 : for (offset = element_size;
6078 : offset < vector_size;
6079 : offset += element_size;)
6080 : {
6081 : Create: s' = extract_field <v_out2, offset>
6082 : Create: s = op <s, s'> // For non SLP cases
6083 : } */
6084 :
6085 237 : if (dump_enabled_p ())
6086 143 : dump_printf_loc (MSG_NOTE, vect_location,
6087 : "Reduce using scalar code.\n");
6088 :
6089 237 : tree compute_type = TREE_TYPE (vectype1);
6090 237 : unsigned element_bitsize = vector_element_bits (vectype1);
6091 237 : unsigned vec_size_in_bits = element_bitsize
6092 237 : * TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6093 237 : tree bitsize = bitsize_int (element_bitsize);
6094 237 : gimple_seq stmts = NULL;
6095 629 : FOR_EACH_VEC_ELT (reduc_inputs, i, vec_temp)
6096 : {
6097 392 : unsigned bit_offset;
6098 784 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, compute_type,
6099 392 : vec_temp, bitsize, bitsize_zero_node);
6100 :
6101 : /* In SLP we don't need to apply reduction operation, so we just
6102 : collect s' values in SCALAR_RESULTS. */
6103 392 : if (slp_reduc)
6104 382 : scalar_results.safe_push (new_temp);
6105 :
6106 392 : for (bit_offset = element_bitsize;
6107 1338 : bit_offset < vec_size_in_bits;
6108 946 : bit_offset += element_bitsize)
6109 : {
6110 946 : tree bitpos = bitsize_int (bit_offset);
6111 946 : new_name = gimple_build (&stmts, BIT_FIELD_REF,
6112 : compute_type, vec_temp,
6113 : bitsize, bitpos);
6114 946 : if (slp_reduc)
6115 : {
6116 : /* In SLP we don't need to apply reduction operation, so
6117 : we just collect s' values in SCALAR_RESULTS. */
6118 936 : new_temp = new_name;
6119 936 : scalar_results.safe_push (new_name);
6120 : }
6121 : else
6122 10 : new_temp = gimple_build (&stmts, code, compute_type,
6123 : new_name, new_temp);
6124 : }
6125 : }
6126 :
6127 : /* The only case where we need to reduce scalar results in a SLP
6128 : reduction, is unrolling. If the size of SCALAR_RESULTS is
6129 : greater than GROUP_SIZE, we reduce them combining elements modulo
6130 : GROUP_SIZE. */
6131 237 : if (slp_reduc)
6132 : {
6133 227 : tree res, first_res, new_res;
6134 :
6135 : /* Reduce multiple scalar results in case of SLP unrolling. */
6136 867 : for (j = group_size; scalar_results.iterate (j, &res);
6137 : j++)
6138 : {
6139 640 : first_res = scalar_results[j % group_size];
6140 640 : new_res = gimple_build (&stmts, code, compute_type,
6141 : first_res, res);
6142 640 : scalar_results[j % group_size] = new_res;
6143 : }
6144 227 : scalar_results.truncate (group_size);
6145 1132 : for (k = 0; k < group_size; k++)
6146 1356 : scalar_results[k] = gimple_convert (&stmts, scalar_type,
6147 678 : scalar_results[k]);
6148 : }
6149 : else
6150 : {
6151 : /* Reduction chain - we have one scalar to keep in
6152 : SCALAR_RESULTS. */
6153 10 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6154 10 : scalar_results.safe_push (new_temp);
6155 : }
6156 :
6157 237 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6158 : }
6159 :
6160 1964 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
6161 0 : && induc_val)
6162 : {
6163 : /* Earlier we set the initial value to be a vector if induc_val
6164 : values. Check the result and if it is induc_val then replace
6165 : with the original initial value, unless induc_val is
6166 : the same as initial_def already. */
6167 0 : tree zcompare = make_ssa_name (boolean_type_node);
6168 0 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
6169 0 : scalar_results[0], induc_val);
6170 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6171 0 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
6172 0 : tree tmp = make_ssa_name (new_scalar_dest);
6173 0 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
6174 0 : initial_def, scalar_results[0]);
6175 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6176 0 : scalar_results[0] = tmp;
6177 : }
6178 : }
6179 :
6180 : /* 2.5 Adjust the final result by the initial value of the reduction
6181 : variable. (When such adjustment is not needed, then
6182 : 'adjustment_def' is zero). For example, if code is PLUS we create:
6183 : new_temp = loop_exit_def + adjustment_def */
6184 :
6185 22176 : if (adjustment_def)
6186 : {
6187 15728 : gcc_assert (!slp_reduc || group_size == 1);
6188 15728 : gimple_seq stmts = NULL;
6189 15728 : if (double_reduc)
6190 : {
6191 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def)));
6192 0 : adjustment_def = gimple_convert (&stmts, vectype, adjustment_def);
6193 0 : new_temp = gimple_build (&stmts, code, vectype,
6194 0 : reduc_inputs[0], adjustment_def);
6195 : }
6196 : else
6197 : {
6198 15728 : new_temp = scalar_results[0];
6199 15728 : gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
6200 15728 : adjustment_def = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6201 : adjustment_def);
6202 15728 : new_temp = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6203 : new_temp);
6204 15728 : new_temp = gimple_build (&stmts, code, TREE_TYPE (compute_vectype),
6205 : new_temp, adjustment_def);
6206 15728 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6207 : }
6208 :
6209 15728 : epilog_stmt = gimple_seq_last_stmt (stmts);
6210 15728 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6211 15728 : scalar_results[0] = new_temp;
6212 : }
6213 :
6214 : /* Record this operation if it could be reused by the epilogue loop. */
6215 22176 : if (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION
6216 22176 : && reduc_inputs.length () == 1)
6217 21971 : loop_vinfo->reusable_accumulators.put (scalar_results[0],
6218 : { orig_reduc_input, reduc_info });
6219 :
6220 : /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
6221 : phis with new adjusted scalar results, i.e., replace use <s_out0>
6222 : with use <s_out4>.
6223 :
6224 : Transform:
6225 : loop_exit:
6226 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6227 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6228 : v_out2 = reduce <v_out1>
6229 : s_out3 = extract_field <v_out2, 0>
6230 : s_out4 = adjust_result <s_out3>
6231 : use <s_out0>
6232 : use <s_out0>
6233 :
6234 : into:
6235 :
6236 : loop_exit:
6237 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6238 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6239 : v_out2 = reduce <v_out1>
6240 : s_out3 = extract_field <v_out2, 0>
6241 : s_out4 = adjust_result <s_out3>
6242 : use <s_out4>
6243 : use <s_out4> */
6244 :
6245 44352 : gcc_assert (live_out_stmts.size () == scalar_results.length ());
6246 22176 : auto_vec<gimple *> phis;
6247 44803 : for (k = 0; k < live_out_stmts.size (); k++)
6248 : {
6249 22627 : stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
6250 22627 : tree scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
6251 :
6252 : /* Find the loop-closed-use at the loop exit of the original scalar
6253 : result. (The reduction result is expected to have two immediate uses,
6254 : one at the latch block, and one at the loop exit). Note with
6255 : early break we can have two exit blocks, so pick the correct PHI. */
6256 92025 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
6257 69398 : if (!is_gimple_debug (USE_STMT (use_p))
6258 69398 : && !flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
6259 : {
6260 22620 : gcc_assert (is_a <gphi *> (USE_STMT (use_p)));
6261 22620 : if (gimple_bb (USE_STMT (use_p)) == loop_exit->dest)
6262 22612 : phis.safe_push (USE_STMT (use_p));
6263 22627 : }
6264 :
6265 45239 : FOR_EACH_VEC_ELT (phis, i, exit_phi)
6266 : {
6267 : /* Replace the uses: */
6268 22612 : orig_name = PHI_RESULT (exit_phi);
6269 :
6270 : /* Look for a single use at the target of the skip edge. */
6271 22612 : if (unify_with_main_loop_p)
6272 : {
6273 30 : use_operand_p use_p;
6274 30 : gimple *user;
6275 30 : if (!single_imm_use (orig_name, &use_p, &user))
6276 0 : gcc_unreachable ();
6277 30 : orig_name = gimple_get_lhs (user);
6278 : }
6279 :
6280 22612 : scalar_result = scalar_results[k];
6281 61183 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
6282 : {
6283 38571 : gphi *use_phi = dyn_cast <gphi *> (use_stmt);
6284 77186 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6285 : {
6286 38593 : if (use_phi
6287 38593 : && (phi_arg_edge_from_use (use_p)->flags & EDGE_ABNORMAL))
6288 : {
6289 0 : gcc_assert (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (orig_name));
6290 0 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (scalar_result) = 1;
6291 : }
6292 38593 : SET_USE (use_p, scalar_result);
6293 : }
6294 38571 : update_stmt (use_stmt);
6295 22612 : }
6296 : }
6297 :
6298 22627 : phis.truncate (0);
6299 : }
6300 22176 : }
6301 :
6302 : /* Return a vector of type VECTYPE that is equal to the vector select
6303 : operation "MASK ? VEC : IDENTITY". Insert the select statements
6304 : before GSI. */
6305 :
6306 : static tree
6307 9 : merge_with_identity (gimple_stmt_iterator *gsi, tree mask, tree vectype,
6308 : tree vec, tree identity)
6309 : {
6310 9 : tree cond = make_temp_ssa_name (vectype, NULL, "cond");
6311 9 : gimple *new_stmt = gimple_build_assign (cond, VEC_COND_EXPR,
6312 : mask, vec, identity);
6313 9 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6314 9 : return cond;
6315 : }
6316 :
6317 : /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
6318 : order, starting with LHS. Insert the extraction statements before GSI and
6319 : associate the new scalar SSA names with variable SCALAR_DEST.
6320 : If MASK is nonzero mask the input and then operate on it unconditionally.
6321 : Return the SSA name for the result. */
6322 :
6323 : static tree
6324 1218 : vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
6325 : tree_code code, tree lhs, tree vector_rhs,
6326 : tree mask)
6327 : {
6328 1218 : tree vectype = TREE_TYPE (vector_rhs);
6329 1218 : tree scalar_type = TREE_TYPE (vectype);
6330 1218 : tree bitsize = TYPE_SIZE (scalar_type);
6331 1218 : unsigned HOST_WIDE_INT vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
6332 1218 : unsigned HOST_WIDE_INT element_bitsize = tree_to_uhwi (bitsize);
6333 :
6334 : /* Re-create a VEC_COND_EXPR to mask the input here in order to be able
6335 : to perform an unconditional element-wise reduction of it. */
6336 1218 : if (mask)
6337 : {
6338 76 : tree masked_vector_rhs = make_temp_ssa_name (vectype, NULL,
6339 : "masked_vector_rhs");
6340 76 : tree neutral_op = neutral_op_for_reduction (scalar_type, code, NULL_TREE,
6341 : false);
6342 76 : tree vector_identity = build_vector_from_val (vectype, neutral_op);
6343 76 : gassign *select = gimple_build_assign (masked_vector_rhs, VEC_COND_EXPR,
6344 : mask, vector_rhs, vector_identity);
6345 76 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6346 76 : vector_rhs = masked_vector_rhs;
6347 : }
6348 :
6349 1218 : for (unsigned HOST_WIDE_INT bit_offset = 0;
6350 5000 : bit_offset < vec_size_in_bits;
6351 3782 : bit_offset += element_bitsize)
6352 : {
6353 3782 : tree bitpos = bitsize_int (bit_offset);
6354 3782 : tree rhs = build3 (BIT_FIELD_REF, scalar_type, vector_rhs,
6355 : bitsize, bitpos);
6356 :
6357 3782 : gassign *stmt = gimple_build_assign (scalar_dest, rhs);
6358 3782 : rhs = make_ssa_name (scalar_dest, stmt);
6359 3782 : gimple_assign_set_lhs (stmt, rhs);
6360 3782 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6361 : /* Fold the vector extract, combining it with a previous reversal
6362 : like seen in PR90579. */
6363 3782 : auto gsi2 = gsi_for_stmt (stmt);
6364 3782 : if (fold_stmt (&gsi2, follow_all_ssa_edges))
6365 354 : update_stmt (gsi_stmt (gsi2));
6366 :
6367 3782 : stmt = gimple_build_assign (scalar_dest, code, lhs, rhs);
6368 3782 : tree new_name = make_ssa_name (scalar_dest, stmt);
6369 3782 : gimple_assign_set_lhs (stmt, new_name);
6370 3782 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6371 3782 : lhs = new_name;
6372 : }
6373 1218 : return lhs;
6374 : }
6375 :
6376 : /* Get a masked internal function equivalent to REDUC_FN. VECTYPE_IN is the
6377 : type of the vector input. */
6378 :
6379 : static internal_fn
6380 3009 : get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
6381 : {
6382 3009 : internal_fn mask_reduc_fn;
6383 3009 : internal_fn mask_len_reduc_fn;
6384 :
6385 3009 : switch (reduc_fn)
6386 : {
6387 0 : case IFN_FOLD_LEFT_PLUS:
6388 0 : mask_reduc_fn = IFN_MASK_FOLD_LEFT_PLUS;
6389 0 : mask_len_reduc_fn = IFN_MASK_LEN_FOLD_LEFT_PLUS;
6390 0 : break;
6391 :
6392 : default:
6393 : return IFN_LAST;
6394 : }
6395 :
6396 0 : if (direct_internal_fn_supported_p (mask_reduc_fn, vectype_in,
6397 : OPTIMIZE_FOR_SPEED))
6398 : return mask_reduc_fn;
6399 0 : if (direct_internal_fn_supported_p (mask_len_reduc_fn, vectype_in,
6400 : OPTIMIZE_FOR_SPEED))
6401 0 : return mask_len_reduc_fn;
6402 : return IFN_LAST;
6403 : }
6404 :
6405 : /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
6406 : statement that sets the live-out value. REDUC_DEF_STMT is the phi
6407 : statement. CODE is the operation performed by STMT_INFO and OPS are
6408 : its scalar operands. REDUC_INDEX is the index of the operand in
6409 : OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
6410 : implements in-order reduction, or IFN_LAST if we should open-code it.
6411 : VECTYPE_IN is the type of the vector input. MASKS specifies the masks
6412 : that should be used to control the operation in a fully-masked loop. */
6413 :
6414 : static bool
6415 913 : vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
6416 : stmt_vec_info stmt_info,
6417 : gimple_stmt_iterator *gsi,
6418 : slp_tree slp_node,
6419 : code_helper code, internal_fn reduc_fn,
6420 : int num_ops, tree vectype_in,
6421 : int reduc_index, vec_loop_masks *masks,
6422 : vec_loop_lens *lens)
6423 : {
6424 913 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
6425 913 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
6426 913 : internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
6427 :
6428 913 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
6429 :
6430 913 : bool is_cond_op = false;
6431 913 : if (!code.is_tree_code ())
6432 : {
6433 29 : code = conditional_internal_fn_code (internal_fn (code));
6434 29 : gcc_assert (code != ERROR_MARK);
6435 : is_cond_op = true;
6436 : }
6437 :
6438 913 : gcc_assert (TREE_CODE_LENGTH (tree_code (code)) == binary_op);
6439 :
6440 913 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out),
6441 : TYPE_VECTOR_SUBPARTS (vectype_in)));
6442 :
6443 : /* ??? We should, when transforming the cycle PHI, record the existing
6444 : scalar def as vector def so looking up the vector def works. This
6445 : would also allow generalizing this for reduction paths of length > 1
6446 : and/or SLP reductions. */
6447 913 : slp_tree reduc_node = SLP_TREE_CHILDREN (slp_node)[reduc_index];
6448 913 : stmt_vec_info reduc_var_def = SLP_TREE_SCALAR_STMTS (reduc_node)[0];
6449 913 : tree reduc_var = gimple_get_lhs (STMT_VINFO_STMT (reduc_var_def));
6450 :
6451 : /* The operands either come from a binary operation or an IFN_COND operation.
6452 : The former is a gimple assign with binary rhs and the latter is a
6453 : gimple call with four arguments. */
6454 913 : gcc_assert (num_ops == 2 || num_ops == 4);
6455 :
6456 913 : auto_vec<tree> vec_oprnds0, vec_opmask;
6457 913 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[(is_cond_op ? 2 : 0)
6458 913 : + (1 - reduc_index)],
6459 : &vec_oprnds0);
6460 : /* For an IFN_COND_OP we also need the vector mask operand. */
6461 913 : if (is_cond_op)
6462 29 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], &vec_opmask);
6463 :
6464 : /* The transform below relies on preserving the original scalar PHI
6465 : and its latch def which we replace. So work backwards from there. */
6466 913 : tree scalar_dest
6467 913 : = gimple_phi_arg_def_from_edge (as_a <gphi *> (STMT_VINFO_STMT
6468 : (reduc_var_def)),
6469 913 : loop_latch_edge (loop));
6470 913 : stmt_vec_info scalar_dest_def_info
6471 913 : = vect_stmt_to_vectorize (loop_vinfo->lookup_def (scalar_dest));
6472 913 : tree scalar_type = TREE_TYPE (scalar_dest);
6473 :
6474 913 : int vec_num = vec_oprnds0.length ();
6475 913 : tree vec_elem_type = TREE_TYPE (vectype_out);
6476 913 : gcc_checking_assert (useless_type_conversion_p (scalar_type, vec_elem_type));
6477 :
6478 913 : tree vector_identity = NULL_TREE;
6479 913 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6480 : {
6481 2 : vector_identity = build_zero_cst (vectype_out);
6482 2 : if (!HONOR_SIGNED_ZEROS (vectype_out))
6483 : ;
6484 : else
6485 : {
6486 2 : gcc_assert (!HONOR_SIGN_DEPENDENT_ROUNDING (vectype_out));
6487 2 : vector_identity = const_unop (NEGATE_EXPR, vectype_out,
6488 : vector_identity);
6489 : }
6490 : }
6491 :
6492 913 : tree scalar_dest_var = vect_create_destination_var (scalar_dest, NULL);
6493 913 : int i;
6494 913 : tree def0;
6495 3044 : FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
6496 : {
6497 1218 : gimple *new_stmt;
6498 1218 : tree mask = NULL_TREE;
6499 1218 : tree len = NULL_TREE;
6500 1218 : tree bias = NULL_TREE;
6501 1218 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6502 : {
6503 9 : tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
6504 : vec_num, vectype_in, i);
6505 9 : if (is_cond_op)
6506 9 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (loop_mask),
6507 9 : loop_mask, vec_opmask[i], gsi);
6508 : else
6509 : mask = loop_mask;
6510 : }
6511 1209 : else if (is_cond_op)
6512 67 : mask = vec_opmask[i];
6513 1218 : if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
6514 : {
6515 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num, vectype_in,
6516 : i, 1, false);
6517 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
6518 0 : bias = build_int_cst (intQI_type_node, biasval);
6519 0 : if (!is_cond_op)
6520 0 : mask = build_minus_one_cst (truth_type_for (vectype_in));
6521 : }
6522 :
6523 : /* Handle MINUS by adding the negative. */
6524 1218 : if (reduc_fn != IFN_LAST && code == MINUS_EXPR)
6525 : {
6526 0 : tree negated = make_ssa_name (vectype_out);
6527 0 : new_stmt = gimple_build_assign (negated, NEGATE_EXPR, def0);
6528 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6529 0 : def0 = negated;
6530 : }
6531 :
6532 9 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
6533 1227 : && mask && mask_reduc_fn == IFN_LAST)
6534 9 : def0 = merge_with_identity (gsi, mask, vectype_out, def0,
6535 : vector_identity);
6536 :
6537 : /* On the first iteration the input is simply the scalar phi
6538 : result, and for subsequent iterations it is the output of
6539 : the preceding operation. */
6540 1218 : if (reduc_fn != IFN_LAST || (mask && mask_reduc_fn != IFN_LAST))
6541 : {
6542 0 : if (mask && len && mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6543 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 5, reduc_var,
6544 : def0, mask, len, bias);
6545 0 : else if (mask && mask_reduc_fn == IFN_MASK_FOLD_LEFT_PLUS)
6546 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 3, reduc_var,
6547 : def0, mask);
6548 : else
6549 0 : new_stmt = gimple_build_call_internal (reduc_fn, 2, reduc_var,
6550 : def0);
6551 : /* For chained SLP reductions the output of the previous reduction
6552 : operation serves as the input of the next. For the final statement
6553 : the output cannot be a temporary - we reuse the original
6554 : scalar destination of the last statement. */
6555 0 : if (i != vec_num - 1)
6556 : {
6557 0 : gimple_set_lhs (new_stmt, scalar_dest_var);
6558 0 : reduc_var = make_ssa_name (scalar_dest_var, new_stmt);
6559 0 : gimple_set_lhs (new_stmt, reduc_var);
6560 : }
6561 : }
6562 : else
6563 : {
6564 1218 : reduc_var = vect_expand_fold_left (gsi, scalar_dest_var,
6565 : tree_code (code), reduc_var, def0,
6566 : mask);
6567 1218 : new_stmt = SSA_NAME_DEF_STMT (reduc_var);
6568 : /* Remove the statement, so that we can use the same code paths
6569 : as for statements that we've just created. */
6570 1218 : gimple_stmt_iterator tmp_gsi = gsi_for_stmt (new_stmt);
6571 1218 : gsi_remove (&tmp_gsi, true);
6572 : }
6573 :
6574 1218 : if (i == vec_num - 1)
6575 : {
6576 913 : gimple_set_lhs (new_stmt, scalar_dest);
6577 913 : vect_finish_replace_stmt (loop_vinfo,
6578 : scalar_dest_def_info,
6579 : new_stmt);
6580 : }
6581 : else
6582 305 : vect_finish_stmt_generation (loop_vinfo,
6583 : scalar_dest_def_info,
6584 : new_stmt, gsi);
6585 :
6586 1218 : slp_node->push_vec_def (new_stmt);
6587 : }
6588 :
6589 913 : return true;
6590 913 : }
6591 :
6592 : /* Function is_nonwrapping_integer_induction.
6593 :
6594 : Check if STMT_VINO (which is part of loop LOOP) both increments and
6595 : does not cause overflow. */
6596 :
6597 : static bool
6598 408 : is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, class loop *loop)
6599 : {
6600 408 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
6601 408 : tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo);
6602 408 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo);
6603 408 : tree lhs_type = TREE_TYPE (gimple_phi_result (phi));
6604 408 : widest_int ni, max_loop_value, lhs_max;
6605 408 : wi::overflow_type overflow = wi::OVF_NONE;
6606 :
6607 : /* Make sure the loop is integer based. */
6608 408 : if (TREE_CODE (base) != INTEGER_CST
6609 109 : || TREE_CODE (step) != INTEGER_CST)
6610 : return false;
6611 :
6612 : /* Check that the max size of the loop will not wrap. */
6613 :
6614 109 : if (TYPE_OVERFLOW_UNDEFINED (lhs_type))
6615 : return true;
6616 :
6617 8 : if (! max_stmt_executions (loop, &ni))
6618 : return false;
6619 :
6620 8 : max_loop_value = wi::mul (wi::to_widest (step), ni, TYPE_SIGN (lhs_type),
6621 8 : &overflow);
6622 8 : if (overflow)
6623 : return false;
6624 :
6625 8 : max_loop_value = wi::add (wi::to_widest (base), max_loop_value,
6626 16 : TYPE_SIGN (lhs_type), &overflow);
6627 8 : if (overflow)
6628 : return false;
6629 :
6630 8 : return (wi::min_precision (max_loop_value, TYPE_SIGN (lhs_type))
6631 8 : <= TYPE_PRECISION (lhs_type));
6632 408 : }
6633 :
6634 : /* Check if masking can be supported by inserting a conditional expression.
6635 : CODE is the code for the operation. COND_FN is the conditional internal
6636 : function, if it exists. VECTYPE_IN is the type of the vector input. */
6637 : static bool
6638 6062 : use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
6639 : tree vectype_in)
6640 : {
6641 6062 : if (cond_fn != IFN_LAST
6642 6062 : && direct_internal_fn_supported_p (cond_fn, vectype_in,
6643 : OPTIMIZE_FOR_SPEED))
6644 : return false;
6645 :
6646 4335 : if (code.is_tree_code ())
6647 4315 : switch (tree_code (code))
6648 : {
6649 376 : case DOT_PROD_EXPR:
6650 376 : case SAD_EXPR:
6651 376 : return true;
6652 :
6653 : default:
6654 : break;
6655 : }
6656 : return false;
6657 : }
6658 :
6659 : /* Insert a conditional expression to enable masked vectorization. CODE is the
6660 : code for the operation. VOP is the array of operands. MASK is the loop
6661 : mask. GSI is a statement iterator used to place the new conditional
6662 : expression. */
6663 : static void
6664 4 : build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
6665 : gimple_stmt_iterator *gsi)
6666 : {
6667 4 : switch (tree_code (code))
6668 : {
6669 4 : case DOT_PROD_EXPR:
6670 4 : {
6671 4 : tree vectype = TREE_TYPE (vop[1]);
6672 4 : tree zero = build_zero_cst (vectype);
6673 4 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6674 4 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6675 : mask, vop[1], zero);
6676 4 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6677 4 : vop[1] = masked_op1;
6678 4 : break;
6679 : }
6680 :
6681 0 : case SAD_EXPR:
6682 0 : {
6683 0 : tree vectype = TREE_TYPE (vop[1]);
6684 0 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6685 0 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6686 : mask, vop[1], vop[0]);
6687 0 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6688 0 : vop[1] = masked_op1;
6689 0 : break;
6690 : }
6691 :
6692 0 : default:
6693 0 : gcc_unreachable ();
6694 : }
6695 4 : }
6696 :
6697 : /* Given an operation with CODE in loop reduction path whose reduction PHI is
6698 : specified by REDUC_INFO, the operation has TYPE of scalar result, and its
6699 : input vectype is represented by VECTYPE_IN. The vectype of vectorized result
6700 : may be different from VECTYPE_IN, either in base type or vectype lanes,
6701 : lane-reducing operation is the case. This function check if it is possible,
6702 : and how to perform partial vectorization on the operation in the context
6703 : of LOOP_VINFO. */
6704 :
6705 : static void
6706 4164 : vect_reduction_update_partial_vector_usage (loop_vec_info loop_vinfo,
6707 : vect_reduc_info reduc_info,
6708 : slp_tree slp_node,
6709 : code_helper code, tree type,
6710 : tree vectype_in)
6711 : {
6712 4164 : enum vect_reduction_type reduc_type = VECT_REDUC_INFO_TYPE (reduc_info);
6713 4164 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
6714 4164 : internal_fn cond_fn
6715 1157 : = ((code.is_internal_fn ()
6716 1157 : && internal_fn_mask_index ((internal_fn)code) != -1)
6717 4164 : ? (internal_fn)code : get_conditional_internal_fn (code, type));
6718 :
6719 4164 : if (reduc_type != FOLD_LEFT_REDUCTION
6720 3395 : && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
6721 7446 : && (cond_fn == IFN_LAST
6722 3282 : || !direct_internal_fn_supported_p (cond_fn, vectype_in,
6723 : OPTIMIZE_FOR_SPEED)))
6724 : {
6725 2068 : if (dump_enabled_p ())
6726 98 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6727 : "can't operate on partial vectors because"
6728 : " no conditional operation is available.\n");
6729 2068 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6730 : }
6731 2096 : else if (reduc_type == FOLD_LEFT_REDUCTION
6732 2096 : && reduc_fn == IFN_LAST
6733 2096 : && !expand_vec_cond_expr_p (vectype_in, truth_type_for (vectype_in)))
6734 : {
6735 0 : if (dump_enabled_p ())
6736 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6737 : "can't operate on partial vectors because"
6738 : " no conditional operation is available.\n");
6739 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6740 : }
6741 2096 : else if (reduc_type == FOLD_LEFT_REDUCTION
6742 769 : && internal_fn_mask_index (reduc_fn) == -1
6743 769 : && FLOAT_TYPE_P (vectype_in)
6744 2865 : && HONOR_SIGN_DEPENDENT_ROUNDING (vectype_in))
6745 : {
6746 0 : if (dump_enabled_p ())
6747 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6748 : "can't operate on partial vectors because"
6749 : " signed zeros cannot be preserved.\n");
6750 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6751 : }
6752 : else
6753 : {
6754 2096 : internal_fn mask_reduc_fn
6755 2096 : = get_masked_reduction_fn (reduc_fn, vectype_in);
6756 2096 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
6757 2096 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
6758 2096 : unsigned nvectors = vect_get_num_copies (loop_vinfo, slp_node);
6759 :
6760 2096 : if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6761 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_in, 1);
6762 : else
6763 2096 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_in, NULL);
6764 : }
6765 4164 : }
6766 :
6767 : /* Check if STMT_INFO is a lane-reducing operation that can be vectorized in
6768 : the context of LOOP_VINFO, and vector cost will be recorded in COST_VEC,
6769 : and the analysis is for slp if SLP_NODE is not NULL.
6770 :
6771 : For a lane-reducing operation, the loop reduction path that it lies in,
6772 : may contain normal operation, or other lane-reducing operation of different
6773 : input type size, an example as:
6774 :
6775 : int sum = 0;
6776 : for (i)
6777 : {
6778 : ...
6779 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
6780 : sum += w[i]; // widen-sum <vector(16) char>
6781 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
6782 : sum += n[i]; // normal <vector(4) int>
6783 : ...
6784 : }
6785 :
6786 : Vectorization factor is essentially determined by operation whose input
6787 : vectype has the most lanes ("vector(16) char" in the example), while we
6788 : need to choose input vectype with the least lanes ("vector(4) int" in the
6789 : example) to determine effective number of vector reduction PHIs. */
6790 :
6791 : bool
6792 403466 : vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
6793 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
6794 : {
6795 403466 : gimple *stmt = stmt_info->stmt;
6796 :
6797 403466 : if (!lane_reducing_stmt_p (stmt))
6798 : return false;
6799 :
6800 722 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
6801 :
6802 722 : if (!INTEGRAL_TYPE_P (type))
6803 : return false;
6804 :
6805 : /* Do not try to vectorize bit-precision reductions. */
6806 722 : if (!type_has_mode_precision_p (type))
6807 : return false;
6808 :
6809 722 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
6810 :
6811 : /* TODO: Support lane-reducing operation that does not directly participate
6812 : in loop reduction. */
6813 722 : if (!reduc_info)
6814 : return false;
6815 :
6816 : /* Lane-reducing pattern inside any inner loop of LOOP_VINFO is not
6817 : recognized. */
6818 722 : gcc_assert (!nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info));
6819 722 : gcc_assert (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION);
6820 :
6821 2888 : for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
6822 : {
6823 2166 : slp_tree slp_op;
6824 2166 : tree op;
6825 2166 : tree vectype;
6826 2166 : enum vect_def_type dt;
6827 :
6828 2166 : if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
6829 : &slp_op, &dt, &vectype))
6830 : {
6831 0 : if (dump_enabled_p ())
6832 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6833 : "use not simple.\n");
6834 0 : return false;
6835 : }
6836 :
6837 2166 : if (!vectype)
6838 : {
6839 6 : vectype = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op),
6840 : slp_op);
6841 6 : if (!vectype)
6842 : return false;
6843 : }
6844 :
6845 2166 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype))
6846 : {
6847 0 : if (dump_enabled_p ())
6848 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6849 : "incompatible vector types for invariants\n");
6850 : return false;
6851 : }
6852 :
6853 2166 : if (i == STMT_VINFO_REDUC_IDX (stmt_info))
6854 722 : continue;
6855 :
6856 : /* There should be at most one cycle def in the stmt. */
6857 1444 : if (VECTORIZABLE_CYCLE_DEF (dt))
6858 : return false;
6859 : }
6860 :
6861 722 : slp_tree node_in = SLP_TREE_CHILDREN (slp_node)[0];
6862 722 : tree vectype_in = SLP_TREE_VECTYPE (node_in);
6863 722 : gcc_assert (vectype_in);
6864 :
6865 : /* Compute number of effective vector statements for costing. */
6866 722 : unsigned int ncopies_for_cost = vect_get_num_copies (loop_vinfo, node_in);
6867 722 : gcc_assert (ncopies_for_cost >= 1);
6868 :
6869 722 : if (vect_is_emulated_mixed_dot_prod (slp_node))
6870 : {
6871 : /* We need extra two invariants: one that contains the minimum signed
6872 : value and one that contains half of its negative. */
6873 15 : int prologue_stmts = 2;
6874 15 : unsigned cost = record_stmt_cost (cost_vec, prologue_stmts,
6875 : scalar_to_vec, slp_node, 0,
6876 : vect_prologue);
6877 15 : if (dump_enabled_p ())
6878 0 : dump_printf (MSG_NOTE, "vectorizable_lane_reducing: "
6879 : "extra prologue_cost = %d .\n", cost);
6880 :
6881 : /* Three dot-products and a subtraction. */
6882 15 : ncopies_for_cost *= 4;
6883 : }
6884 :
6885 722 : record_stmt_cost (cost_vec, (int) ncopies_for_cost, vector_stmt, slp_node,
6886 : 0, vect_body);
6887 :
6888 722 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
6889 : {
6890 113 : enum tree_code code = gimple_assign_rhs_code (stmt);
6891 113 : vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
6892 113 : node_in, code, type,
6893 : vectype_in);
6894 : }
6895 :
6896 : /* Transform via vect_transform_reduction. */
6897 722 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
6898 722 : return true;
6899 : }
6900 :
6901 : /* Function vectorizable_reduction.
6902 :
6903 : Check if STMT_INFO performs a reduction operation that can be vectorized.
6904 : If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6905 : stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6906 : Return true if STMT_INFO is vectorizable in this way.
6907 :
6908 : This function also handles reduction idioms (patterns) that have been
6909 : recognized in advance during vect_pattern_recog. In this case, STMT_INFO
6910 : may be of this form:
6911 : X = pattern_expr (arg0, arg1, ..., X)
6912 : and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
6913 : sequence that had been detected and replaced by the pattern-stmt
6914 : (STMT_INFO).
6915 :
6916 : This function also handles reduction of condition expressions, for example:
6917 : for (int i = 0; i < N; i++)
6918 : if (a[i] < value)
6919 : last = a[i];
6920 : This is handled by vectorising the loop and creating an additional vector
6921 : containing the loop indexes for which "a[i] < value" was true. In the
6922 : function epilogue this is reduced to a single max value and then used to
6923 : index into the vector of results.
6924 :
6925 : In some cases of reduction patterns, the type of the reduction variable X is
6926 : different than the type of the other arguments of STMT_INFO.
6927 : In such cases, the vectype that is used when transforming STMT_INFO into
6928 : a vector stmt is different than the vectype that is used to determine the
6929 : vectorization factor, because it consists of a different number of elements
6930 : than the actual number of elements that are being operated upon in parallel.
6931 :
6932 : For example, consider an accumulation of shorts into an int accumulator.
6933 : On some targets it's possible to vectorize this pattern operating on 8
6934 : shorts at a time (hence, the vectype for purposes of determining the
6935 : vectorization factor should be V8HI); on the other hand, the vectype that
6936 : is used to create the vector form is actually V4SI (the type of the result).
6937 :
6938 : Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
6939 : indicates what is the actual level of parallelism (V8HI in the example), so
6940 : that the right vectorization factor would be derived. This vectype
6941 : corresponds to the type of arguments to the reduction stmt, and should *NOT*
6942 : be used to create the vectorized stmt. The right vectype for the vectorized
6943 : stmt is obtained from the type of the result X:
6944 : get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
6945 :
6946 : This means that, contrary to "regular" reductions (or "regular" stmts in
6947 : general), the following equation:
6948 : STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
6949 : does *NOT* necessarily hold for reduction patterns. */
6950 :
6951 : bool
6952 402744 : vectorizable_reduction (loop_vec_info loop_vinfo,
6953 : stmt_vec_info stmt_info, slp_tree slp_node,
6954 : slp_instance slp_node_instance,
6955 : stmt_vector_for_cost *cost_vec)
6956 : {
6957 402744 : tree vectype_in = NULL_TREE;
6958 402744 : enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
6959 402744 : stmt_vec_info cond_stmt_vinfo = NULL;
6960 402744 : int i;
6961 402744 : int ncopies;
6962 402744 : bool single_defuse_cycle = false;
6963 402744 : tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE;
6964 402744 : tree cond_reduc_val = NULL_TREE;
6965 :
6966 : /* Make sure it was already recognized as a reduction computation. */
6967 402744 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
6968 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def
6969 402744 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
6970 : return false;
6971 :
6972 : /* The reduction meta. */
6973 84956 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
6974 :
6975 84956 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
6976 : {
6977 1490 : gcc_assert (is_a <gphi *> (stmt_info->stmt));
6978 : /* We eventually need to set a vector type on invariant arguments. */
6979 : unsigned j;
6980 : slp_tree child;
6981 4462 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
6982 2980 : if (!vect_maybe_update_slp_op_vectype (child,
6983 : SLP_TREE_VECTYPE (slp_node)))
6984 : {
6985 0 : if (dump_enabled_p ())
6986 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6987 : "incompatible vector types for "
6988 : "invariants\n");
6989 : return false;
6990 : }
6991 2980 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
6992 2980 : && !useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
6993 : SLP_TREE_VECTYPE (child)))
6994 : {
6995 : /* With bools we can have mask and non-mask precision vectors
6996 : or different non-mask precisions. while pattern recog is
6997 : supposed to guarantee consistency here, we do not have
6998 : pattern stmts for PHIs (PR123316).
6999 : Deal with that here instead of ICEing later. */
7000 8 : if (dump_enabled_p ())
7001 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7002 : "incompatible vector type setup from "
7003 : "bool pattern detection\n");
7004 : return false;
7005 : }
7006 : /* Analysis for double-reduction is done on the outer
7007 : loop PHI, nested cycles have no further restrictions. */
7008 1482 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7009 1482 : return true;
7010 : }
7011 :
7012 83466 : if (!is_a <gphi *> (stmt_info->stmt))
7013 : {
7014 7965 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def);
7015 7965 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
7016 7965 : return true;
7017 : }
7018 :
7019 75501 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
7020 75501 : stmt_vec_info phi_info = stmt_info;
7021 75501 : bool double_reduc = false;
7022 75501 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
7023 : {
7024 : /* We arrive here for both the inner loop LC PHI and the
7025 : outer loop PHI. The latter is what we want to analyze the
7026 : reduction with. The LC PHI is handled by vectorizable_lc_phi. */
7027 322 : if (gimple_bb (stmt_info->stmt) != loop->header)
7028 0 : return false;
7029 :
7030 : /* Set loop and phi_info to the inner loop. */
7031 322 : use_operand_p use_p;
7032 322 : gimple *use_stmt;
7033 322 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
7034 : &use_p, &use_stmt);
7035 322 : gcc_assert (res);
7036 322 : phi_info = loop_vinfo->lookup_stmt (use_stmt);
7037 322 : loop = loop->inner;
7038 322 : double_reduc = true;
7039 : }
7040 :
7041 75501 : const bool reduc_chain = reduc_info->is_reduc_chain;
7042 75501 : slp_node_instance->reduc_phis = slp_node;
7043 : /* ??? We're leaving slp_node to point to the PHIs, we only
7044 : need it to get at the number of vector stmts which wasn't
7045 : yet initialized for the instance root. */
7046 :
7047 : /* PHIs should not participate in patterns. */
7048 75501 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7049 75501 : gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
7050 :
7051 : /* Verify following REDUC_IDX from the latch def leads us back to the PHI
7052 : and compute the reduction chain length. Discover the real
7053 : reduction operation stmt on the way (slp_for_stmt_info). */
7054 75501 : unsigned reduc_chain_length = 0;
7055 75501 : stmt_info = NULL;
7056 75501 : slp_tree slp_for_stmt_info = NULL;
7057 75501 : slp_tree vdef_slp = slp_node_instance->root;
7058 166779 : while (vdef_slp != slp_node)
7059 : {
7060 92370 : int reduc_idx = SLP_TREE_REDUC_IDX (vdef_slp);
7061 92370 : if (reduc_idx == -1)
7062 : {
7063 1084 : if (dump_enabled_p ())
7064 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7065 : "reduction chain broken by patterns.\n");
7066 1092 : return false;
7067 : }
7068 91286 : stmt_vec_info vdef = SLP_TREE_REPRESENTATIVE (vdef_slp);
7069 91286 : if (is_a <gphi *> (vdef->stmt))
7070 : {
7071 644 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7072 : /* Do not count PHIs towards the chain length. */
7073 644 : continue;
7074 : }
7075 90642 : gimple_match_op op;
7076 90642 : if (!gimple_extract_op (vdef->stmt, &op))
7077 : {
7078 0 : if (dump_enabled_p ())
7079 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7080 : "reduction chain includes unsupported"
7081 : " statement type.\n");
7082 : return false;
7083 : }
7084 90642 : if (CONVERT_EXPR_CODE_P (op.code))
7085 : {
7086 5390 : if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
7087 : {
7088 8 : if (dump_enabled_p ())
7089 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7090 : "conversion in the reduction chain.\n");
7091 : return false;
7092 : }
7093 5382 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
7094 : }
7095 : else
7096 : {
7097 : /* First non-conversion stmt. */
7098 85252 : if (!slp_for_stmt_info)
7099 74409 : slp_for_stmt_info = vdef_slp;
7100 :
7101 85252 : if (lane_reducing_op_p (op.code))
7102 : {
7103 : /* The last operand of lane-reducing operation is for
7104 : reduction. */
7105 722 : gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
7106 :
7107 722 : slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
7108 722 : tree vectype_op = SLP_TREE_VECTYPE (op_node);
7109 722 : tree type_op = TREE_TYPE (op.ops[0]);
7110 722 : if (!vectype_op)
7111 : {
7112 9 : vectype_op = get_vectype_for_scalar_type (loop_vinfo,
7113 : type_op);
7114 9 : if (!vectype_op
7115 9 : || !vect_maybe_update_slp_op_vectype (op_node,
7116 : vectype_op))
7117 : return false;
7118 : }
7119 :
7120 : /* To accommodate lane-reducing operations of mixed input
7121 : vectypes, choose input vectype with the least lanes for the
7122 : reduction PHI statement, which would result in the most
7123 : ncopies for vectorized reduction results. */
7124 722 : if (!vectype_in
7125 722 : || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
7126 751 : < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
7127 : vectype_in = vectype_op;
7128 : }
7129 84530 : else if (!vectype_in)
7130 73716 : vectype_in = SLP_TREE_VECTYPE (slp_node);
7131 85252 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7132 : }
7133 90634 : reduc_chain_length++;
7134 : }
7135 74409 : if (!slp_for_stmt_info)
7136 : {
7137 0 : if (dump_enabled_p ())
7138 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7139 : "only noop-conversions in the reduction chain.\n");
7140 : return false;
7141 : }
7142 74409 : stmt_info = SLP_TREE_REPRESENTATIVE (slp_for_stmt_info);
7143 :
7144 : /* PHIs should not participate in patterns. */
7145 74409 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7146 :
7147 : /* 1. Is vectorizable reduction? */
7148 : /* Not supportable if the reduction variable is used in the loop, unless
7149 : it's a reduction chain. */
7150 74409 : if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer
7151 0 : && !reduc_chain)
7152 : return false;
7153 :
7154 : /* Reductions that are not used even in an enclosing outer-loop,
7155 : are expected to be "live" (used out of the loop). */
7156 74409 : if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
7157 0 : && !STMT_VINFO_LIVE_P (stmt_info))
7158 : return false;
7159 :
7160 : /* 2. Has this been recognized as a reduction pattern?
7161 :
7162 : Check if STMT represents a pattern that has been recognized
7163 : in earlier analysis stages. For stmts that represent a pattern,
7164 : the STMT_VINFO_RELATED_STMT field records the last stmt in
7165 : the original sequence that constitutes the pattern. */
7166 :
7167 74409 : stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
7168 74409 : if (orig_stmt_info)
7169 : {
7170 5102 : gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
7171 5102 : gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
7172 : }
7173 :
7174 : /* 3. Check the operands of the operation. The first operands are defined
7175 : inside the loop body. The last operand is the reduction variable,
7176 : which is defined by the loop-header-phi. */
7177 :
7178 74409 : tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
7179 74409 : VECT_REDUC_INFO_VECTYPE (reduc_info) = vectype_out;
7180 :
7181 74409 : gimple_match_op op;
7182 74409 : if (!gimple_extract_op (stmt_info->stmt, &op))
7183 0 : gcc_unreachable ();
7184 74409 : bool lane_reducing = lane_reducing_op_p (op.code);
7185 :
7186 74409 : if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
7187 22154 : && !SCALAR_FLOAT_TYPE_P (op.type))
7188 : return false;
7189 :
7190 : /* Do not try to vectorize bit-precision reductions. */
7191 74409 : if (!type_has_mode_precision_p (op.type)
7192 1764 : && op.code != BIT_AND_EXPR
7193 1629 : && op.code != BIT_IOR_EXPR
7194 74885 : && op.code != BIT_XOR_EXPR)
7195 : return false;
7196 :
7197 : /* Lane-reducing ops also never can be used in a SLP reduction group
7198 : since we'll mix lanes belonging to different reductions. But it's
7199 : OK to use them in a reduction chain or when the reduction group
7200 : has just one element. */
7201 74099 : if (lane_reducing
7202 74099 : && !reduc_chain
7203 656 : && SLP_TREE_LANES (slp_node) > 1)
7204 : {
7205 0 : if (dump_enabled_p ())
7206 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7207 : "lane-reducing reduction in reduction group.\n");
7208 : return false;
7209 : }
7210 :
7211 : /* We'll verify the reduction operation only later - avoid
7212 : all operations that mismatch on the number of SLP children. */
7213 148198 : if (op.num_ops != SLP_TREE_CHILDREN (slp_for_stmt_info).length ())
7214 : {
7215 0 : if (dump_enabled_p ())
7216 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7217 : "unsupported reduction operation.\n");
7218 : return false;
7219 : }
7220 :
7221 : /* All uses but the last are expected to be defined in the loop.
7222 : The last use is the reduction variable. In case of nested cycle this
7223 : assumption is not true: we use reduc_index to record the index of the
7224 : reduction variable. */
7225 74099 : slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
7226 74099 : tree *vectype_op = XALLOCAVEC (tree, op.num_ops);
7227 74099 : gcc_assert (op.code != COND_EXPR || !COMPARISON_CLASS_P (op.ops[0]));
7228 237359 : for (i = 0; i < (int) op.num_ops; i++)
7229 : {
7230 : /* The condition of COND_EXPR is checked in vectorizable_condition(). */
7231 163260 : if (i == 0 && op.code == COND_EXPR)
7232 81712 : continue;
7233 :
7234 162406 : stmt_vec_info def_stmt_info;
7235 162406 : enum vect_def_type dt;
7236 162406 : if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
7237 : i, &op.ops[i], &slp_op[i], &dt,
7238 162406 : &vectype_op[i], &def_stmt_info))
7239 : {
7240 0 : if (dump_enabled_p ())
7241 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7242 : "use not simple.\n");
7243 0 : return false;
7244 : }
7245 :
7246 : /* Skip reduction operands, and for an IFN_COND_OP we might hit the
7247 : reduction operand twice (once as definition, once as else). */
7248 162406 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]
7249 324812 : == SLP_TREE_CHILDREN
7250 162406 : (slp_for_stmt_info)[SLP_TREE_REDUC_IDX (slp_for_stmt_info)])
7251 80858 : continue;
7252 :
7253 : /* There should be only one cycle def in the stmt, the one
7254 : leading to reduc_def. */
7255 81548 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]->cycle_info.id != -1)
7256 : return false;
7257 :
7258 81548 : if (!vectype_op[i])
7259 7358 : vectype_op[i]
7260 7358 : = get_vectype_for_scalar_type (loop_vinfo,
7261 7358 : TREE_TYPE (op.ops[i]), slp_op[i]);
7262 :
7263 : /* Record how the non-reduction-def value of COND_EXPR is defined.
7264 : ??? For a chain of multiple CONDs we'd have to match them up all. */
7265 81548 : if (op.code == COND_EXPR && reduc_chain_length == 1)
7266 : {
7267 831 : if (dt == vect_constant_def)
7268 : {
7269 118 : cond_reduc_dt = dt;
7270 118 : cond_reduc_val = op.ops[i];
7271 : }
7272 713 : else if (dt == vect_induction_def
7273 408 : && def_stmt_info
7274 1121 : && is_nonwrapping_integer_induction (def_stmt_info, loop))
7275 : {
7276 109 : cond_reduc_dt = dt;
7277 109 : cond_stmt_vinfo = def_stmt_info;
7278 : }
7279 : }
7280 : }
7281 :
7282 74099 : enum vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7283 : /* If we have a condition reduction, see if we can simplify it further. */
7284 74099 : if (reduction_type == COND_REDUCTION)
7285 : {
7286 842 : if (SLP_TREE_LANES (slp_node) != 1)
7287 : return false;
7288 :
7289 : /* When the condition uses the reduction value in the condition, fail. */
7290 818 : if (SLP_TREE_REDUC_IDX (slp_node) == 0)
7291 : {
7292 0 : if (dump_enabled_p ())
7293 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7294 : "condition depends on previous iteration\n");
7295 : return false;
7296 : }
7297 :
7298 818 : if (reduc_chain_length == 1
7299 818 : && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in,
7300 : OPTIMIZE_FOR_SPEED)
7301 795 : || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
7302 : vectype_in,
7303 : OPTIMIZE_FOR_SPEED)))
7304 : {
7305 0 : if (dump_enabled_p ())
7306 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7307 : "optimizing condition reduction with"
7308 : " FOLD_EXTRACT_LAST.\n");
7309 0 : VECT_REDUC_INFO_TYPE (reduc_info) = EXTRACT_LAST_REDUCTION;
7310 : }
7311 818 : else if (cond_reduc_dt == vect_induction_def)
7312 : {
7313 109 : tree base
7314 : = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo);
7315 109 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo);
7316 :
7317 109 : gcc_assert (TREE_CODE (base) == INTEGER_CST
7318 : && TREE_CODE (step) == INTEGER_CST);
7319 109 : cond_reduc_val = NULL_TREE;
7320 109 : enum tree_code cond_reduc_op_code = ERROR_MARK;
7321 109 : tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
7322 109 : if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
7323 : ;
7324 : /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
7325 : above base; punt if base is the minimum value of the type for
7326 : MAX_EXPR or maximum value of the type for MIN_EXPR for now. */
7327 97 : else if (tree_int_cst_sgn (step) == -1)
7328 : {
7329 18 : cond_reduc_op_code = MIN_EXPR;
7330 18 : if (tree_int_cst_sgn (base) == -1)
7331 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7332 18 : else if (tree_int_cst_lt (base,
7333 18 : TYPE_MAX_VALUE (TREE_TYPE (base))))
7334 18 : cond_reduc_val
7335 18 : = int_const_binop (PLUS_EXPR, base, integer_one_node);
7336 : }
7337 : else
7338 : {
7339 79 : cond_reduc_op_code = MAX_EXPR;
7340 79 : if (tree_int_cst_sgn (base) == 1)
7341 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7342 79 : else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base)),
7343 : base))
7344 79 : cond_reduc_val
7345 79 : = int_const_binop (MINUS_EXPR, base, integer_one_node);
7346 : }
7347 97 : if (cond_reduc_val)
7348 : {
7349 97 : if (dump_enabled_p ())
7350 61 : dump_printf_loc (MSG_NOTE, vect_location,
7351 : "condition expression based on "
7352 : "integer induction.\n");
7353 97 : VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_op_code;
7354 97 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info)
7355 97 : = cond_reduc_val;
7356 97 : VECT_REDUC_INFO_TYPE (reduc_info) = INTEGER_INDUC_COND_REDUCTION;
7357 : }
7358 : }
7359 709 : else if (cond_reduc_dt == vect_constant_def)
7360 : {
7361 108 : enum vect_def_type cond_initial_dt;
7362 108 : tree cond_initial_val = vect_phi_initial_value (reduc_def_phi);
7363 108 : vect_is_simple_use (cond_initial_val, loop_vinfo, &cond_initial_dt);
7364 108 : if (cond_initial_dt == vect_constant_def
7365 133 : && types_compatible_p (TREE_TYPE (cond_initial_val),
7366 25 : TREE_TYPE (cond_reduc_val)))
7367 : {
7368 25 : tree e = fold_binary (LE_EXPR, boolean_type_node,
7369 : cond_initial_val, cond_reduc_val);
7370 25 : if (e && (integer_onep (e) || integer_zerop (e)))
7371 : {
7372 25 : if (dump_enabled_p ())
7373 16 : dump_printf_loc (MSG_NOTE, vect_location,
7374 : "condition expression based on "
7375 : "compile time constant.\n");
7376 : /* Record reduction code at analysis stage. */
7377 25 : VECT_REDUC_INFO_CODE (reduc_info)
7378 25 : = integer_onep (e) ? MAX_EXPR : MIN_EXPR;
7379 25 : VECT_REDUC_INFO_TYPE (reduc_info) = CONST_COND_REDUCTION;
7380 : }
7381 : }
7382 : }
7383 : }
7384 :
7385 74075 : if (STMT_VINFO_LIVE_P (phi_info))
7386 : return false;
7387 :
7388 74075 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
7389 :
7390 74075 : gcc_assert (ncopies >= 1);
7391 :
7392 74075 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
7393 :
7394 : /* 4.2. Check support for the epilog operation.
7395 :
7396 : If STMT represents a reduction pattern, then the type of the
7397 : reduction variable may be different than the type of the rest
7398 : of the arguments. For example, consider the case of accumulation
7399 : of shorts into an int accumulator; The original code:
7400 : S1: int_a = (int) short_a;
7401 : orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
7402 :
7403 : was replaced with:
7404 : STMT: int_acc = widen_sum <short_a, int_acc>
7405 :
7406 : This means that:
7407 : 1. The tree-code that is used to create the vector operation in the
7408 : epilog code (that reduces the partial results) is not the
7409 : tree-code of STMT, but is rather the tree-code of the original
7410 : stmt from the pattern that STMT is replacing. I.e, in the example
7411 : above we want to use 'widen_sum' in the loop, but 'plus' in the
7412 : epilog.
7413 : 2. The type (mode) we use to check available target support
7414 : for the vector operation to be created in the *epilog*, is
7415 : determined by the type of the reduction variable (in the example
7416 : above we'd check this: optab_handler (plus_optab, vect_int_mode])).
7417 : However the type (mode) we use to check available target support
7418 : for the vector operation to be created *inside the loop*, is
7419 : determined by the type of the other arguments to STMT (in the
7420 : example we'd check this: optab_handler (widen_sum_optab,
7421 : vect_short_mode)).
7422 :
7423 : This is contrary to "regular" reductions, in which the types of all
7424 : the arguments are the same as the type of the reduction variable.
7425 : For "regular" reductions we can therefore use the same vector type
7426 : (and also the same tree-code) when generating the epilog code and
7427 : when generating the code inside the loop. */
7428 :
7429 74075 : code_helper orig_code = VECT_REDUC_INFO_CODE (reduc_info);
7430 :
7431 : /* If conversion might have created a conditional operation like
7432 : IFN_COND_ADD already. Use the internal code for the following checks. */
7433 74075 : if (orig_code.is_internal_fn ())
7434 : {
7435 6835 : tree_code new_code = conditional_internal_fn_code (internal_fn (orig_code));
7436 6835 : orig_code = new_code != ERROR_MARK ? new_code : orig_code;
7437 : }
7438 :
7439 74075 : VECT_REDUC_INFO_CODE (reduc_info) = orig_code;
7440 :
7441 74075 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7442 74075 : if (reduction_type == TREE_CODE_REDUCTION)
7443 : {
7444 : /* Check whether it's ok to change the order of the computation.
7445 : Generally, when vectorizing a reduction we change the order of the
7446 : computation. This may change the behavior of the program in some
7447 : cases, so we need to check that this is ok. One exception is when
7448 : vectorizing an outer-loop: the inner-loop is executed sequentially,
7449 : and therefore vectorizing reductions in the inner-loop during
7450 : outer-loop vectorization is safe. Likewise when we are vectorizing
7451 : a series of reductions using SLP and the VF is one the reductions
7452 : are performed in scalar order. */
7453 73257 : if (!reduc_chain
7454 73257 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
7455 : ;
7456 73100 : else if (needs_fold_left_reduction_p (op.type, orig_code))
7457 : {
7458 : /* When vectorizing a reduction chain w/o SLP the reduction PHI
7459 : is not directly used in stmt. */
7460 5201 : if (reduc_chain_length != 1)
7461 : {
7462 83 : if (dump_enabled_p ())
7463 20 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7464 : "in-order reduction chain without SLP.\n");
7465 : return false;
7466 : }
7467 : /* Code generation doesn't support function calls other
7468 : than .COND_*. */
7469 5118 : if (!op.code.is_tree_code ()
7470 5306 : && !(op.code.is_internal_fn ()
7471 94 : && conditional_internal_fn_code (internal_fn (op.code))
7472 : != ERROR_MARK))
7473 : {
7474 18 : if (dump_enabled_p ())
7475 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7476 : "in-order reduction chain operation not "
7477 : "supported.\n");
7478 : return false;
7479 : }
7480 5100 : VECT_REDUC_INFO_TYPE (reduc_info)
7481 5100 : = reduction_type = FOLD_LEFT_REDUCTION;
7482 : }
7483 67899 : else if (!commutative_binary_op_p (orig_code, op.type)
7484 67899 : || !associative_binary_op_p (orig_code, op.type))
7485 : {
7486 172 : if (dump_enabled_p ())
7487 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7488 : "reduction: not commutative/associative\n");
7489 : return false;
7490 : }
7491 : }
7492 :
7493 5100 : if ((reduction_type == COND_REDUCTION
7494 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7495 : || reduction_type == CONST_COND_REDUCTION
7496 68702 : || reduction_type == EXTRACT_LAST_REDUCTION)
7497 818 : && ncopies > 1)
7498 : {
7499 276 : if (dump_enabled_p ())
7500 60 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7501 : "multiple types in condition reduction.\n");
7502 : return false;
7503 : }
7504 :
7505 : /* See if we can convert a mask vector to a corresponding bool data vector
7506 : to perform the epilogue reduction. */
7507 73526 : tree alt_vectype_out = NULL_TREE;
7508 73526 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out))
7509 : {
7510 1156 : alt_vectype_out
7511 2312 : = get_related_vectype_for_scalar_type (loop_vinfo->vector_mode,
7512 1156 : TREE_TYPE (vectype_out),
7513 : TYPE_VECTOR_SUBPARTS
7514 : (vectype_out));
7515 1156 : if (!alt_vectype_out
7516 1156 : || maybe_ne (TYPE_VECTOR_SUBPARTS (alt_vectype_out),
7517 2312 : TYPE_VECTOR_SUBPARTS (vectype_out))
7518 2312 : || !expand_vec_cond_expr_p (alt_vectype_out, vectype_out))
7519 : alt_vectype_out = NULL_TREE;
7520 : }
7521 :
7522 73526 : internal_fn reduc_fn = IFN_LAST;
7523 73526 : if (reduction_type == TREE_CODE_REDUCTION
7524 73526 : || reduction_type == FOLD_LEFT_REDUCTION
7525 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7526 542 : || reduction_type == CONST_COND_REDUCTION)
7527 : {
7528 67998 : if (reduction_type == FOLD_LEFT_REDUCTION
7529 77412 : ? fold_left_reduction_fn (orig_code, &reduc_fn)
7530 67998 : : reduction_fn_for_scalar_code (orig_code, &reduc_fn))
7531 : {
7532 72426 : internal_fn sbool_fn = IFN_LAST;
7533 72426 : if (reduc_fn == IFN_LAST)
7534 : ;
7535 70326 : else if ((!VECTOR_BOOLEAN_TYPE_P (vectype_out)
7536 1156 : || (GET_MODE_CLASS (TYPE_MODE (vectype_out))
7537 : == MODE_VECTOR_BOOL))
7538 139496 : && direct_internal_fn_supported_p (reduc_fn, vectype_out,
7539 : OPTIMIZE_FOR_SPEED))
7540 : ;
7541 18525 : else if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
7542 1156 : && sbool_reduction_fn_for_fn (reduc_fn, &sbool_fn)
7543 19681 : && direct_internal_fn_supported_p (sbool_fn, vectype_out,
7544 : OPTIMIZE_FOR_SPEED))
7545 131 : reduc_fn = sbool_fn;
7546 18394 : else if (reduction_type != FOLD_LEFT_REDUCTION
7547 18394 : && alt_vectype_out
7548 18394 : && direct_internal_fn_supported_p (reduc_fn, alt_vectype_out,
7549 : OPTIMIZE_FOR_SPEED))
7550 804 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7551 : else
7552 : {
7553 17590 : if (dump_enabled_p ())
7554 946 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7555 : "reduc op not supported by target.\n");
7556 :
7557 17590 : reduc_fn = IFN_LAST;
7558 : }
7559 : }
7560 : else
7561 : {
7562 672 : if (dump_enabled_p ())
7563 48 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7564 : "no reduc code for scalar code.\n");
7565 :
7566 : return false;
7567 : }
7568 72426 : if (reduc_fn == IFN_LAST
7569 72426 : && VECTOR_BOOLEAN_TYPE_P (vectype_out))
7570 : {
7571 221 : if (!alt_vectype_out)
7572 : {
7573 12 : if (dump_enabled_p ())
7574 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7575 : "cannot turn mask into bool data vector for "
7576 : "reduction epilogue.\n");
7577 : return false;
7578 : }
7579 209 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7580 : }
7581 : }
7582 428 : else if (reduction_type == COND_REDUCTION)
7583 : {
7584 428 : int scalar_precision
7585 428 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
7586 428 : cr_index_scalar_type = make_unsigned_type (scalar_precision);
7587 428 : cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
7588 : vectype_out);
7589 :
7590 428 : if (direct_internal_fn_supported_p (IFN_REDUC_MAX, cr_index_vector_type,
7591 : OPTIMIZE_FOR_SPEED))
7592 22 : reduc_fn = IFN_REDUC_MAX;
7593 : }
7594 72842 : VECT_REDUC_INFO_FN (reduc_info) = reduc_fn;
7595 :
7596 72842 : if (reduction_type != EXTRACT_LAST_REDUCTION
7597 : && reduc_fn == IFN_LAST
7598 : && !nunits_out.is_constant ())
7599 : {
7600 : if (dump_enabled_p ())
7601 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7602 : "missing target support for reduction on"
7603 : " variable-length vectors.\n");
7604 : return false;
7605 : }
7606 :
7607 : /* For SLP reductions, see if there is a neutral value we can use. */
7608 72842 : tree neutral_op = NULL_TREE;
7609 72842 : tree initial_value = NULL_TREE;
7610 72842 : if (reduc_chain)
7611 2246 : initial_value = vect_phi_initial_value (reduc_def_phi);
7612 72842 : neutral_op = neutral_op_for_reduction (TREE_TYPE
7613 : (gimple_phi_result (reduc_def_phi)),
7614 : orig_code, initial_value);
7615 72842 : VECT_REDUC_INFO_NEUTRAL_OP (reduc_info) = neutral_op;
7616 :
7617 72842 : if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
7618 : {
7619 : /* We can't support in-order reductions of code such as this:
7620 :
7621 : for (int i = 0; i < n1; ++i)
7622 : for (int j = 0; j < n2; ++j)
7623 : l += a[j];
7624 :
7625 : since GCC effectively transforms the loop when vectorizing:
7626 :
7627 : for (int i = 0; i < n1 / VF; ++i)
7628 : for (int j = 0; j < n2; ++j)
7629 : for (int k = 0; k < VF; ++k)
7630 : l += a[j];
7631 :
7632 : which is a reassociation of the original operation. */
7633 66 : if (dump_enabled_p ())
7634 20 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7635 : "in-order double reduction not supported.\n");
7636 :
7637 : return false;
7638 : }
7639 :
7640 72776 : if (reduction_type == FOLD_LEFT_REDUCTION
7641 4362 : && SLP_TREE_LANES (slp_node) > 1
7642 159 : && !reduc_chain)
7643 : {
7644 : /* We cannot use in-order reductions in this case because there is
7645 : an implicit reassociation of the operations involved. */
7646 61 : if (dump_enabled_p ())
7647 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7648 : "in-order unchained SLP reductions not supported.\n");
7649 : return false;
7650 : }
7651 :
7652 : /* For double reductions, and for SLP reductions with a neutral value,
7653 : we construct a variable-length initial vector by loading a vector
7654 : full of the neutral value and then shift-and-inserting the start
7655 : values into the low-numbered elements. This is however not needed
7656 : when neutral and initial value are equal or we can handle the
7657 : initial value via adjustment in the epilogue. */
7658 72715 : if ((double_reduc || neutral_op)
7659 : && !nunits_out.is_constant ()
7660 : && reduction_type != INTEGER_INDUC_COND_REDUCTION
7661 : && !((SLP_TREE_LANES (slp_node) == 1 || reduc_chain)
7662 : && neutral_op
7663 : && (!double_reduc
7664 : || operand_equal_p (neutral_op,
7665 : vect_phi_initial_value (reduc_def_phi))))
7666 : && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT,
7667 : vectype_out, OPTIMIZE_FOR_BOTH))
7668 : {
7669 : if (dump_enabled_p ())
7670 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7671 : "reduction on variable-length vectors requires"
7672 : " target support for a vector-shift-and-insert"
7673 : " operation.\n");
7674 : return false;
7675 : }
7676 :
7677 : /* Check extra constraints for variable-length unchained SLP reductions. */
7678 72715 : if (!reduc_chain
7679 : && !nunits_out.is_constant ())
7680 : {
7681 : /* We checked above that we could build the initial vector when
7682 : there's a neutral element value. Check here for the case in
7683 : which each SLP statement has its own initial value and in which
7684 : that value needs to be repeated for every instance of the
7685 : statement within the initial vector. */
7686 : unsigned int group_size = SLP_TREE_LANES (slp_node);
7687 : if (!neutral_op
7688 : && !can_duplicate_and_interleave_p (loop_vinfo, group_size,
7689 : TREE_TYPE (vectype_out)))
7690 : {
7691 : if (dump_enabled_p ())
7692 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7693 : "unsupported form of SLP reduction for"
7694 : " variable-length vectors: cannot build"
7695 : " initial vector.\n");
7696 : return false;
7697 : }
7698 : /* The epilogue code relies on the number of elements being a multiple
7699 : of the group size. The duplicate-and-interleave approach to setting
7700 : up the initial vector does too. */
7701 : if (!multiple_p (nunits_out, group_size))
7702 : {
7703 : if (dump_enabled_p ())
7704 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7705 : "unsupported form of SLP reduction for"
7706 : " variable-length vectors: the vector size"
7707 : " is not a multiple of the number of results.\n");
7708 : return false;
7709 : }
7710 : }
7711 :
7712 72715 : if (reduction_type == COND_REDUCTION)
7713 : {
7714 428 : widest_int ni;
7715 :
7716 428 : if (! max_loop_iterations (loop, &ni))
7717 : {
7718 14 : if (dump_enabled_p ())
7719 0 : dump_printf_loc (MSG_NOTE, vect_location,
7720 : "loop count not known, cannot create cond "
7721 : "reduction.\n");
7722 : return false;
7723 : }
7724 : /* Convert backedges to iterations. */
7725 414 : ni += 1;
7726 :
7727 : /* The additional index will be the same type as the condition. Check
7728 : that the loop can fit into this less one (because we'll use up the
7729 : zero slot for when there are no matches). */
7730 414 : tree max_index = TYPE_MAX_VALUE (cr_index_scalar_type);
7731 414 : if (wi::geu_p (ni, wi::to_widest (max_index)))
7732 : {
7733 90 : if (dump_enabled_p ())
7734 54 : dump_printf_loc (MSG_NOTE, vect_location,
7735 : "loop size is greater than data size.\n");
7736 : return false;
7737 : }
7738 428 : }
7739 :
7740 : /* In case the vectorization factor (VF) is bigger than the number
7741 : of elements that we can fit in a vectype (nunits), we have to generate
7742 : more than one vector stmt - i.e - we need to "unroll" the
7743 : vector stmt by a factor VF/nunits. For more details see documentation
7744 : in vectorizable_operation. */
7745 :
7746 : /* If the reduction is used in an outer loop we need to generate
7747 : VF intermediate results, like so (e.g. for ncopies=2):
7748 : r0 = phi (init, r0)
7749 : r1 = phi (init, r1)
7750 : r0 = x0 + r0;
7751 : r1 = x1 + r1;
7752 : (i.e. we generate VF results in 2 registers).
7753 : In this case we have a separate def-use cycle for each copy, and therefore
7754 : for each copy we get the vector def for the reduction variable from the
7755 : respective phi node created for this copy.
7756 :
7757 : Otherwise (the reduction is unused in the loop nest), we can combine
7758 : together intermediate results, like so (e.g. for ncopies=2):
7759 : r = phi (init, r)
7760 : r = x0 + r;
7761 : r = x1 + r;
7762 : (i.e. we generate VF/2 results in a single register).
7763 : In this case for each copy we get the vector def for the reduction variable
7764 : from the vectorized reduction operation generated in the previous iteration.
7765 :
7766 : This only works when we see both the reduction PHI and its only consumer
7767 : in vectorizable_reduction and there are no intermediate stmts
7768 : participating. When unrolling we want each unrolled iteration to have its
7769 : own reduction accumulator since one of the main goals of unrolling a
7770 : reduction is to reduce the aggregate loop-carried latency. */
7771 72611 : if (ncopies > 1
7772 72611 : && !reduc_chain
7773 8143 : && SLP_TREE_LANES (slp_node) == 1
7774 7974 : && (STMT_VINFO_RELEVANT (stmt_info) <= vect_used_only_live)
7775 7951 : && reduc_chain_length == 1
7776 7537 : && loop_vinfo->suggested_unroll_factor == 1)
7777 72611 : single_defuse_cycle = true;
7778 :
7779 72611 : if (single_defuse_cycle && !lane_reducing)
7780 : {
7781 6583 : gcc_assert (op.code != COND_EXPR);
7782 :
7783 : /* 4. check support for the operation in the loop
7784 :
7785 : This isn't necessary for the lane reduction codes, since they
7786 : can only be produced by pattern matching, and it's up to the
7787 : pattern matcher to test for support. The main reason for
7788 : specifically skipping this step is to avoid rechecking whether
7789 : mixed-sign dot-products can be implemented using signed
7790 : dot-products. */
7791 6583 : machine_mode vec_mode = TYPE_MODE (vectype_in);
7792 6583 : if (!directly_supported_p (op.code, vectype_in, optab_vector))
7793 : {
7794 2077 : if (dump_enabled_p ())
7795 36 : dump_printf (MSG_NOTE, "op not supported by target.\n");
7796 4154 : if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
7797 2077 : || !vect_can_vectorize_without_simd_p (op.code))
7798 : single_defuse_cycle = false;
7799 : else
7800 11 : if (dump_enabled_p ())
7801 0 : dump_printf (MSG_NOTE, "proceeding using word mode.\n");
7802 : }
7803 :
7804 6583 : if (vect_emulated_vector_p (vectype_in)
7805 6583 : && !vect_can_vectorize_without_simd_p (op.code))
7806 : {
7807 0 : if (dump_enabled_p ())
7808 0 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
7809 : return false;
7810 : }
7811 : }
7812 72611 : if (dump_enabled_p () && single_defuse_cycle)
7813 704 : dump_printf_loc (MSG_NOTE, vect_location,
7814 : "using single def-use cycle for reduction by reducing "
7815 : "multiple vectors to one in the loop body\n");
7816 72611 : VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info) = single_defuse_cycle;
7817 :
7818 : /* For lane-reducing operation, the below processing related to single
7819 : defuse-cycle will be done in its own vectorizable function. One more
7820 : thing to note is that the operation must not be involved in fold-left
7821 : reduction. */
7822 72611 : single_defuse_cycle &= !lane_reducing;
7823 :
7824 72611 : if (single_defuse_cycle || reduction_type == FOLD_LEFT_REDUCTION)
7825 28559 : for (i = 0; i < (int) op.num_ops; i++)
7826 19832 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_op[i]))
7827 : {
7828 0 : if (dump_enabled_p ())
7829 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7830 : "incompatible vector types for invariants\n");
7831 : return false;
7832 : }
7833 :
7834 72611 : vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
7835 : reduction_type, ncopies, cost_vec);
7836 : /* Cost the reduction op inside the loop if transformed via
7837 : vect_transform_reduction for non-lane-reducing operation. Otherwise
7838 : this is costed by the separate vectorizable_* routines. */
7839 72611 : if (single_defuse_cycle)
7840 4517 : record_stmt_cost (cost_vec, ncopies, vector_stmt,
7841 : slp_for_stmt_info, 0, vect_body);
7842 :
7843 72611 : if (dump_enabled_p ()
7844 72611 : && reduction_type == FOLD_LEFT_REDUCTION)
7845 262 : dump_printf_loc (MSG_NOTE, vect_location,
7846 : "using an in-order (fold-left) reduction.\n");
7847 72611 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7848 :
7849 : /* All but single defuse-cycle optimized and fold-left reductions go
7850 : through their own vectorizable_* routines. */
7851 72611 : stmt_vec_info tem
7852 72611 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (slp_node_instance));
7853 72611 : if (!single_defuse_cycle && reduction_type != FOLD_LEFT_REDUCTION)
7854 63884 : STMT_VINFO_DEF_TYPE (tem) = vect_internal_def;
7855 : else
7856 : {
7857 8727 : STMT_VINFO_DEF_TYPE (tem) = vect_reduction_def;
7858 8727 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
7859 4051 : vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
7860 : slp_node, op.code, op.type,
7861 : vectype_in);
7862 : }
7863 : return true;
7864 : }
7865 :
7866 : /* STMT_INFO is a dot-product reduction whose multiplication operands
7867 : have different signs. Emit a sequence to emulate the operation
7868 : using a series of signed DOT_PROD_EXPRs and return the last
7869 : statement generated. VEC_DEST is the result of the vector operation
7870 : and VOP lists its inputs. */
7871 :
7872 : static gassign *
7873 4 : vect_emulate_mixed_dot_prod (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
7874 : gimple_stmt_iterator *gsi, tree vec_dest,
7875 : tree vop[3])
7876 : {
7877 4 : tree wide_vectype = signed_type_for (TREE_TYPE (vec_dest));
7878 4 : tree narrow_vectype = signed_type_for (TREE_TYPE (vop[0]));
7879 4 : tree narrow_elttype = TREE_TYPE (narrow_vectype);
7880 4 : gimple *new_stmt;
7881 :
7882 : /* Make VOP[0] the unsigned operand VOP[1] the signed operand. */
7883 4 : if (!TYPE_UNSIGNED (TREE_TYPE (vop[0])))
7884 0 : std::swap (vop[0], vop[1]);
7885 :
7886 : /* Convert all inputs to signed types. */
7887 12 : for (int i = 1; i < 3; ++i)
7888 8 : if (TYPE_UNSIGNED (TREE_TYPE (vop[i])))
7889 : {
7890 0 : tree tmp = make_ssa_name (signed_type_for (TREE_TYPE (vop[i])));
7891 0 : new_stmt = gimple_build_assign (tmp, NOP_EXPR, vop[i]);
7892 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7893 0 : vop[i] = tmp;
7894 : }
7895 :
7896 : /* In the comments below we assume 8-bit inputs for simplicity,
7897 : but the approach works for any full integer type. */
7898 :
7899 : /* Create a vector of -128. */
7900 4 : tree min_narrow_elttype = TYPE_MIN_VALUE (narrow_elttype);
7901 4 : tree min_narrow = build_vector_from_val (TREE_TYPE (vop[0]),
7902 4 : fold_convert
7903 : (TREE_TYPE (TREE_TYPE (vop[0])),
7904 : min_narrow_elttype));
7905 :
7906 : /* Create a vector of 64. */
7907 4 : auto half_wi = wi::lrshift (wi::to_wide (min_narrow_elttype), 1);
7908 4 : tree half_narrow = wide_int_to_tree (narrow_elttype, half_wi);
7909 4 : half_narrow = build_vector_from_val (narrow_vectype, half_narrow);
7910 :
7911 : /* Emit: SUB_RES = VOP[0] - 128 in an unsigned type. */
7912 4 : tree sub_res = make_ssa_name (TREE_TYPE (vop[0]));
7913 4 : new_stmt = gimple_build_assign (sub_res, PLUS_EXPR, vop[0], min_narrow);
7914 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7915 :
7916 4 : vop[0] = make_ssa_name (narrow_vectype);
7917 4 : new_stmt = gimple_build_assign (vop[0], VIEW_CONVERT_EXPR,
7918 : build1 (VIEW_CONVERT_EXPR, narrow_vectype,
7919 : sub_res));
7920 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7921 :
7922 : /* Emit:
7923 :
7924 : STAGE1 = DOT_PROD_EXPR <VOP[1], 64, VOP[2]>;
7925 : STAGE2 = DOT_PROD_EXPR <VOP[1], 64, STAGE1>;
7926 : STAGE3 = DOT_PROD_EXPR <SUB_RES, -128, STAGE2>;
7927 :
7928 : on the basis that x * y == (x - 128) * y + 64 * y + 64 * y
7929 : Doing the two 64 * y steps first allows more time to compute x. */
7930 4 : tree stage1 = make_ssa_name (wide_vectype);
7931 4 : new_stmt = gimple_build_assign (stage1, DOT_PROD_EXPR,
7932 : vop[1], half_narrow, vop[2]);
7933 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7934 :
7935 4 : tree stage2 = make_ssa_name (wide_vectype);
7936 4 : new_stmt = gimple_build_assign (stage2, DOT_PROD_EXPR,
7937 : vop[1], half_narrow, stage1);
7938 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7939 :
7940 4 : tree stage3 = make_ssa_name (wide_vectype);
7941 4 : new_stmt = gimple_build_assign (stage3, DOT_PROD_EXPR,
7942 : vop[0], vop[1], stage2);
7943 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7944 :
7945 : /* Convert STAGE3 to the reduction type. */
7946 4 : return gimple_build_assign (vec_dest, CONVERT_EXPR, stage3);
7947 4 : }
7948 :
7949 : /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
7950 : value. */
7951 :
7952 : bool
7953 2667 : vect_transform_reduction (loop_vec_info loop_vinfo,
7954 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
7955 : slp_tree slp_node)
7956 : {
7957 2667 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
7958 2667 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
7959 2667 : unsigned vec_num;
7960 :
7961 2667 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
7962 :
7963 2667 : if (nested_in_vect_loop_p (loop, stmt_info))
7964 : {
7965 0 : loop = loop->inner;
7966 0 : gcc_assert (VECT_REDUC_INFO_DEF_TYPE (reduc_info)
7967 : == vect_double_reduction_def);
7968 : }
7969 :
7970 2667 : gimple_match_op op;
7971 2667 : if (!gimple_extract_op (stmt_info->stmt, &op))
7972 0 : gcc_unreachable ();
7973 :
7974 : /* All uses but the last are expected to be defined in the loop.
7975 : The last use is the reduction variable. In case of nested cycle this
7976 : assumption is not true: we use reduc_index to record the index of the
7977 : reduction variable. */
7978 2667 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
7979 2667 : tree vectype_in = SLP_TREE_VECTYPE (slp_node);
7980 2667 : if (lane_reducing_op_p (op.code))
7981 263 : vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]);
7982 :
7983 2667 : vec_num = vect_get_num_copies (loop_vinfo, SLP_TREE_CHILDREN (slp_node)[0]);
7984 :
7985 2667 : code_helper code = canonicalize_code (op.code, op.type);
7986 2667 : internal_fn cond_fn
7987 482 : = ((code.is_internal_fn ()
7988 482 : && internal_fn_mask_index ((internal_fn)code) != -1)
7989 2667 : ? (internal_fn)code : get_conditional_internal_fn (code, op.type));
7990 :
7991 2667 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
7992 2667 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
7993 2667 : bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
7994 :
7995 : /* Transform. */
7996 2667 : tree new_temp = NULL_TREE;
7997 18669 : auto_vec<tree> vec_oprnds[3];
7998 :
7999 2667 : if (dump_enabled_p ())
8000 771 : dump_printf_loc (MSG_NOTE, vect_location, "transform reduction.\n");
8001 :
8002 : /* A binary COND_OP reduction must have the same definition and else
8003 : value. */
8004 3149 : bool cond_fn_p = code.is_internal_fn ()
8005 482 : && conditional_internal_fn_code (internal_fn (code)) != ERROR_MARK;
8006 482 : if (cond_fn_p)
8007 : {
8008 482 : gcc_assert (code == IFN_COND_ADD || code == IFN_COND_SUB
8009 : || code == IFN_COND_MUL || code == IFN_COND_AND
8010 : || code == IFN_COND_IOR || code == IFN_COND_XOR
8011 : || code == IFN_COND_MIN || code == IFN_COND_MAX);
8012 482 : gcc_assert (op.num_ops == 4
8013 : && (op.ops[reduc_index]
8014 : == op.ops[internal_fn_else_index ((internal_fn) code)]));
8015 : }
8016 :
8017 2667 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
8018 :
8019 2667 : vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
8020 2667 : if (reduction_type == FOLD_LEFT_REDUCTION)
8021 : {
8022 913 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
8023 913 : gcc_assert (code.is_tree_code () || cond_fn_p);
8024 913 : return vectorize_fold_left_reduction
8025 913 : (loop_vinfo, stmt_info, gsi, slp_node,
8026 913 : code, reduc_fn, op.num_ops, vectype_in,
8027 913 : reduc_index, masks, lens);
8028 : }
8029 :
8030 1754 : bool single_defuse_cycle = VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info);
8031 1754 : bool lane_reducing = lane_reducing_op_p (code);
8032 1491 : gcc_assert (single_defuse_cycle || lane_reducing);
8033 :
8034 1754 : if (lane_reducing)
8035 : {
8036 : /* The last operand of lane-reducing op is for reduction. */
8037 263 : gcc_assert (reduc_index == (int) op.num_ops - 1);
8038 : }
8039 :
8040 : /* Create the destination vector */
8041 1754 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
8042 1754 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
8043 :
8044 : /* Get NCOPIES vector definitions for all operands except the reduction
8045 : definition. */
8046 1754 : if (!cond_fn_p)
8047 : {
8048 1301 : gcc_assert (reduc_index >= 0 && reduc_index <= 2);
8049 1301 : vect_get_vec_defs (loop_vinfo, slp_node,
8050 1301 : single_defuse_cycle && reduc_index == 0
8051 1301 : ? NULL_TREE : op.ops[0], &vec_oprnds[0],
8052 1301 : single_defuse_cycle && reduc_index == 1
8053 1301 : ? NULL_TREE : op.ops[1], &vec_oprnds[1],
8054 1301 : op.num_ops == 3
8055 263 : && !(single_defuse_cycle && reduc_index == 2)
8056 1393 : ? op.ops[2] : NULL_TREE, &vec_oprnds[2]);
8057 : }
8058 : else
8059 : {
8060 : /* For a conditional operation pass the truth type as mask
8061 : vectype. */
8062 453 : gcc_assert (single_defuse_cycle
8063 : && (reduc_index == 1 || reduc_index == 2));
8064 453 : vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
8065 : &vec_oprnds[0],
8066 2 : reduc_index == 1 ? NULL_TREE : op.ops[1],
8067 : &vec_oprnds[1],
8068 453 : reduc_index == 2 ? NULL_TREE : op.ops[2],
8069 : &vec_oprnds[2]);
8070 : }
8071 :
8072 : /* For single def-use cycles get one copy of the vectorized reduction
8073 : definition. */
8074 1754 : if (single_defuse_cycle)
8075 : {
8076 1662 : vect_get_vec_defs (loop_vinfo, slp_node,
8077 1662 : reduc_index == 0 ? op.ops[0] : NULL_TREE,
8078 : &vec_oprnds[0],
8079 1662 : reduc_index == 1 ? op.ops[1] : NULL_TREE,
8080 : &vec_oprnds[1],
8081 1662 : reduc_index == 2 ? op.ops[2] : NULL_TREE,
8082 : &vec_oprnds[2]);
8083 : }
8084 92 : else if (lane_reducing)
8085 : {
8086 : /* For normal reduction, consistency between vectorized def/use is
8087 : naturally ensured when mapping from scalar statement. But if lane-
8088 : reducing op is involved in reduction, thing would become somewhat
8089 : complicated in that the op's result and operand for accumulation are
8090 : limited to less lanes than other operands, which certainly causes
8091 : def/use mismatch on adjacent statements around the op if do not have
8092 : any kind of specific adjustment. One approach is to refit lane-
8093 : reducing op in the way of introducing new trivial pass-through copies
8094 : to fix possible def/use gap, so as to make it behave like a normal op.
8095 : And vector reduction PHIs are always generated to the full extent, no
8096 : matter lane-reducing op exists or not. If some copies or PHIs are
8097 : actually superfluous, they would be cleaned up by passes after
8098 : vectorization. An example for single-lane slp, lane-reducing ops
8099 : with mixed input vectypes in a reduction chain, is given as below.
8100 : Similarly, this handling is applicable for multiple-lane slp as well.
8101 :
8102 : int sum = 1;
8103 : for (i)
8104 : {
8105 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
8106 : sum += w[i]; // widen-sum <vector(16) char>
8107 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
8108 : sum += n[i]; // normal <vector(4) int>
8109 : }
8110 :
8111 : The vector size is 128-bit，vectorization factor is 16. Reduction
8112 : statements would be transformed as:
8113 :
8114 : vector<4> int sum_v0 = { 0, 0, 0, 1 };
8115 : vector<4> int sum_v1 = { 0, 0, 0, 0 };
8116 : vector<4> int sum_v2 = { 0, 0, 0, 0 };
8117 : vector<4> int sum_v3 = { 0, 0, 0, 0 };
8118 :
8119 : for (i / 16)
8120 : {
8121 : sum_v0 = DOT_PROD (d0_v0[i: 0 ~ 15], d1_v0[i: 0 ~ 15], sum_v0);
8122 : sum_v1 = sum_v1; // copy
8123 : sum_v2 = sum_v2; // copy
8124 : sum_v3 = sum_v3; // copy
8125 :
8126 : sum_v0 = sum_v0; // copy
8127 : sum_v1 = WIDEN_SUM (w_v1[i: 0 ~ 15], sum_v1);
8128 : sum_v2 = sum_v2; // copy
8129 : sum_v3 = sum_v3; // copy
8130 :
8131 : sum_v0 = sum_v0; // copy
8132 : sum_v1 = SAD (s0_v1[i: 0 ~ 7 ], s1_v1[i: 0 ~ 7 ], sum_v1);
8133 : sum_v2 = SAD (s0_v2[i: 8 ~ 15], s1_v2[i: 8 ~ 15], sum_v2);
8134 : sum_v3 = sum_v3; // copy
8135 :
8136 : sum_v0 += n_v0[i: 0 ~ 3 ];
8137 : sum_v1 += n_v1[i: 4 ~ 7 ];
8138 : sum_v2 += n_v2[i: 8 ~ 11];
8139 : sum_v3 += n_v3[i: 12 ~ 15];
8140 : }
8141 :
8142 : Moreover, for a higher instruction parallelism in final vectorized
8143 : loop, it is considered to make those effective vector lane-reducing
8144 : ops be distributed evenly among all def-use cycles. In the above
8145 : example, DOT_PROD, WIDEN_SUM and SADs are generated into disparate
8146 : cycles, instruction dependency among them could be eliminated. */
8147 92 : unsigned effec_ncopies = vec_oprnds[0].length ();
8148 92 : unsigned total_ncopies = vec_oprnds[reduc_index].length ();
8149 :
8150 92 : gcc_assert (effec_ncopies <= total_ncopies);
8151 :
8152 92 : if (effec_ncopies < total_ncopies)
8153 : {
8154 276 : for (unsigned i = 0; i < op.num_ops - 1; i++)
8155 : {
8156 368 : gcc_assert (vec_oprnds[i].length () == effec_ncopies);
8157 184 : vec_oprnds[i].safe_grow_cleared (total_ncopies);
8158 : }
8159 : }
8160 :
8161 92 : tree reduc_vectype_in = vectype_in;
8162 92 : gcc_assert (reduc_vectype_in);
8163 :
8164 92 : unsigned effec_reduc_ncopies
8165 92 : = vect_get_num_copies (loop_vinfo, SLP_TREE_CHILDREN (slp_node)[0]);
8166 :
8167 92 : gcc_assert (effec_ncopies <= effec_reduc_ncopies);
8168 :
8169 92 : if (effec_ncopies < effec_reduc_ncopies)
8170 : {
8171 : /* Find suitable def-use cycles to generate vectorized statements
8172 : into, and reorder operands based on the selection. */
8173 0 : unsigned curr_pos = VECT_REDUC_INFO_RESULT_POS (reduc_info);
8174 0 : unsigned next_pos = (curr_pos + effec_ncopies) % effec_reduc_ncopies;
8175 :
8176 0 : gcc_assert (curr_pos < effec_reduc_ncopies);
8177 0 : VECT_REDUC_INFO_RESULT_POS (reduc_info) = next_pos;
8178 :
8179 0 : if (curr_pos)
8180 : {
8181 0 : unsigned count = effec_reduc_ncopies - effec_ncopies;
8182 0 : unsigned start = curr_pos - count;
8183 :
8184 0 : if ((int) start < 0)
8185 : {
8186 0 : count = curr_pos;
8187 0 : start = 0;
8188 : }
8189 :
8190 0 : for (unsigned i = 0; i < op.num_ops - 1; i++)
8191 : {
8192 0 : for (unsigned j = effec_ncopies; j > start; j--)
8193 : {
8194 0 : unsigned k = j - 1;
8195 0 : std::swap (vec_oprnds[i][k], vec_oprnds[i][k + count]);
8196 0 : gcc_assert (!vec_oprnds[i][k]);
8197 : }
8198 : }
8199 : }
8200 : }
8201 : }
8202 :
8203 1754 : bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (slp_node);
8204 3026 : unsigned num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
8205 1754 : unsigned mask_index = 0;
8206 :
8207 7723 : for (unsigned i = 0; i < num; ++i)
8208 : {
8209 5969 : gimple *new_stmt;
8210 5969 : tree vop[3] = { vec_oprnds[0][i], vec_oprnds[1][i], NULL_TREE };
8211 5969 : if (!vop[0] || !vop[1])
8212 : {
8213 485 : tree reduc_vop = vec_oprnds[reduc_index][i];
8214 :
8215 : /* If could not generate an effective vector statement for current
8216 : portion of reduction operand, insert a trivial copy to simply
8217 : handle over the operand to other dependent statements. */
8218 485 : gcc_assert (reduc_vop);
8219 :
8220 485 : if (TREE_CODE (reduc_vop) == SSA_NAME
8221 485 : && !SSA_NAME_IS_DEFAULT_DEF (reduc_vop))
8222 485 : new_stmt = SSA_NAME_DEF_STMT (reduc_vop);
8223 : else
8224 : {
8225 0 : new_temp = make_ssa_name (vec_dest);
8226 0 : new_stmt = gimple_build_assign (new_temp, reduc_vop);
8227 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
8228 : gsi);
8229 : }
8230 : }
8231 5484 : else if (masked_loop_p && !mask_by_cond_expr)
8232 : {
8233 : /* No conditional ifns have been defined for lane-reducing op
8234 : yet. */
8235 16 : gcc_assert (!lane_reducing);
8236 :
8237 16 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8238 : vec_num, vectype_in,
8239 : mask_index++);
8240 16 : gcall *call;
8241 24 : if (code.is_internal_fn () && cond_fn_p)
8242 : {
8243 16 : gcc_assert (op.num_ops >= 3
8244 : && internal_fn_mask_index (internal_fn (code)) == 0);
8245 8 : vop[2] = vec_oprnds[2][i];
8246 8 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask),
8247 : mask, vop[0], gsi);
8248 8 : call = gimple_build_call_internal (cond_fn, 4, mask, vop[1],
8249 : vop[2], vop[reduc_index]);
8250 : }
8251 : else
8252 8 : call = gimple_build_call_internal (cond_fn, 4, mask, vop[0],
8253 : vop[1], vop[reduc_index]);
8254 16 : new_temp = make_ssa_name (vec_dest, call);
8255 16 : gimple_call_set_lhs (call, new_temp);
8256 16 : gimple_call_set_nothrow (call, true);
8257 16 : vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi);
8258 16 : new_stmt = call;
8259 : }
8260 : else
8261 : {
8262 5468 : if (op.num_ops >= 3)
8263 1774 : vop[2] = vec_oprnds[2][i];
8264 :
8265 5468 : if (masked_loop_p && mask_by_cond_expr)
8266 : {
8267 4 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8268 : vec_num, vectype_in,
8269 : mask_index++);
8270 4 : build_vect_cond_expr (code, vop, mask, gsi);
8271 : }
8272 :
8273 5468 : if (emulated_mixed_dot_prod)
8274 4 : new_stmt = vect_emulate_mixed_dot_prod (loop_vinfo, stmt_info, gsi,
8275 : vec_dest, vop);
8276 :
8277 6806 : else if (code.is_internal_fn () && !cond_fn_p)
8278 0 : new_stmt = gimple_build_call_internal (internal_fn (code),
8279 : op.num_ops,
8280 : vop[0], vop[1], vop[2]);
8281 6806 : else if (code.is_internal_fn () && cond_fn_p)
8282 1342 : new_stmt = gimple_build_call_internal (internal_fn (code),
8283 : op.num_ops,
8284 : vop[0], vop[1], vop[2],
8285 : vop[reduc_index]);
8286 : else
8287 4122 : new_stmt = gimple_build_assign (vec_dest, tree_code (op.code),
8288 : vop[0], vop[1], vop[2]);
8289 5468 : new_temp = make_ssa_name (vec_dest, new_stmt);
8290 5468 : gimple_set_lhs (new_stmt, new_temp);
8291 5468 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8292 : }
8293 :
8294 5969 : if (single_defuse_cycle && i < num - 1)
8295 3571 : vec_oprnds[reduc_index].safe_push (gimple_get_lhs (new_stmt));
8296 : else
8297 2398 : slp_node->push_vec_def (new_stmt);
8298 : }
8299 :
8300 : return true;
8301 10668 : }
8302 :
8303 : /* Transform phase of a cycle PHI. */
8304 :
8305 : bool
8306 23775 : vect_transform_cycle_phi (loop_vec_info loop_vinfo,
8307 : stmt_vec_info stmt_info,
8308 : slp_tree slp_node, slp_instance slp_node_instance)
8309 : {
8310 23775 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
8311 23775 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
8312 23775 : int i;
8313 23775 : bool nested_cycle = false;
8314 23775 : int vec_num;
8315 :
8316 23775 : if (nested_in_vect_loop_p (loop, stmt_info))
8317 : {
8318 23775 : loop = loop->inner;
8319 23775 : nested_cycle = true;
8320 : }
8321 :
8322 23775 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
8323 23775 : if (reduc_info
8324 23113 : && (VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
8325 23113 : || VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION))
8326 : /* Leave the scalar phi in place. */
8327 : return true;
8328 :
8329 22200 : if (reduc_info && reduc_info->is_reduc_chain && dump_enabled_p ())
8330 127 : dump_printf_loc (MSG_NOTE, vect_location,
8331 : "vectorizing a reduction chain\n");
8332 :
8333 22862 : vec_num = vect_get_num_copies (loop_vinfo, slp_node);
8334 :
8335 : /* Check whether we should use a single PHI node and accumulate
8336 : vectors to one before the backedge. */
8337 22862 : if (reduc_info && VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info))
8338 22862 : vec_num = 1;
8339 :
8340 : /* Create the destination vector */
8341 22862 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
8342 22862 : tree vec_dest = vect_create_destination_var (gimple_phi_result (phi),
8343 : vectype_out);
8344 :
8345 : /* Get the loop-entry arguments. */
8346 22862 : auto_vec<tree> vec_initial_defs;
8347 22862 : vec_initial_defs.reserve (vec_num);
8348 : /* Optimize: if initial_def is for REDUC_MAX smaller than the base
8349 : and we can't use zero for induc_val, use initial_def. Similarly
8350 : for REDUC_MIN and initial_def larger than the base. */
8351 22862 : if (reduc_info
8352 22200 : && VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
8353 : {
8354 62 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8355 62 : tree initial_def = vect_phi_initial_value (phi);
8356 62 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).safe_push (initial_def);
8357 62 : tree induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
8358 62 : if (TREE_CODE (initial_def) == INTEGER_CST
8359 60 : && !integer_zerop (induc_val)
8360 122 : && ((VECT_REDUC_INFO_CODE (reduc_info) == MAX_EXPR
8361 42 : && tree_int_cst_lt (initial_def, induc_val))
8362 58 : || (VECT_REDUC_INFO_CODE (reduc_info) == MIN_EXPR
8363 18 : && tree_int_cst_lt (induc_val, initial_def))))
8364 : {
8365 2 : induc_val = initial_def;
8366 : /* Communicate we used the initial_def to epilouge
8367 : generation. */
8368 2 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
8369 : }
8370 62 : vec_initial_defs.quick_push
8371 62 : (build_vector_from_val (vectype_out, induc_val));
8372 62 : }
8373 22800 : else if (nested_cycle)
8374 : {
8375 748 : unsigned phi_idx = loop_preheader_edge (loop)->dest_idx;
8376 748 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[phi_idx],
8377 : &vec_initial_defs);
8378 : }
8379 : else
8380 : {
8381 22052 : gcc_assert (slp_node == slp_node_instance->reduc_phis);
8382 22052 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
8383 22052 : vec<stmt_vec_info> &stmts = SLP_TREE_SCALAR_STMTS (slp_node);
8384 :
8385 22052 : unsigned int num_phis = stmts.length ();
8386 22052 : if (reduc_info->is_reduc_chain)
8387 203 : num_phis = 1;
8388 22052 : initial_values.reserve (num_phis);
8389 66602 : for (unsigned int i = 0; i < num_phis; ++i)
8390 : {
8391 22498 : gphi *this_phi = as_a<gphi *> (stmts[i]->stmt);
8392 22498 : initial_values.quick_push (vect_phi_initial_value (this_phi));
8393 : }
8394 22052 : tree neutral_op = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
8395 22052 : if (vec_num == 1
8396 22052 : && vect_find_reusable_accumulator (loop_vinfo,
8397 : reduc_info, vectype_out))
8398 : ;
8399 : /* Try to simplify the vector initialization by applying an
8400 : adjustment after the reduction has been performed. This
8401 : can also break a critical path but on the other hand
8402 : requires to keep the initial value live across the loop. */
8403 17944 : else if (neutral_op
8404 17361 : && initial_values.length () == 1
8405 17176 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
8406 35043 : && !operand_equal_p (neutral_op, initial_values[0]))
8407 : {
8408 12148 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info)
8409 12148 : = initial_values[0];
8410 12148 : initial_values[0] = neutral_op;
8411 : }
8412 22052 : if (!VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
8413 4108 : || loop_vinfo->main_loop_edge)
8414 43658 : get_initial_defs_for_reduction (loop_vinfo, reduc_info, vectype_out,
8415 : &vec_initial_defs, vec_num,
8416 : stmts.length (), neutral_op);
8417 : }
8418 :
8419 22862 : if (reduc_info)
8420 22200 : if (auto *accumulator = VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
8421 : {
8422 4108 : tree def = accumulator->reduc_input;
8423 4108 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8424 : {
8425 4105 : unsigned int nreduc;
8426 8210 : bool res = constant_multiple_p (TYPE_VECTOR_SUBPARTS
8427 4105 : (TREE_TYPE (def)),
8428 4105 : TYPE_VECTOR_SUBPARTS (vectype_out),
8429 : &nreduc);
8430 0 : gcc_assert (res);
8431 4105 : gimple_seq stmts = NULL;
8432 : /* Reduce the single vector to a smaller one. */
8433 4105 : if (nreduc != 1)
8434 : {
8435 : /* Perform the reduction in the appropriate type. */
8436 4105 : tree rvectype = vectype_out;
8437 4105 : if (!useless_type_conversion_p (TREE_TYPE (vectype_out),
8438 4105 : TREE_TYPE (TREE_TYPE (def))))
8439 235 : rvectype = build_vector_type (TREE_TYPE (TREE_TYPE (def)),
8440 : TYPE_VECTOR_SUBPARTS
8441 470 : (vectype_out));
8442 4105 : def = vect_create_partial_epilog (def, rvectype,
8443 : VECT_REDUC_INFO_CODE
8444 : (reduc_info),
8445 : &stmts);
8446 : }
8447 : /* The epilogue loop might use a different vector mode, like
8448 : VNx2DI vs. V2DI. */
8449 4105 : if (TYPE_MODE (vectype_out) != TYPE_MODE (TREE_TYPE (def)))
8450 : {
8451 0 : tree reduc_type = build_vector_type_for_mode
8452 0 : (TREE_TYPE (TREE_TYPE (def)), TYPE_MODE (vectype_out));
8453 0 : def = gimple_convert (&stmts, reduc_type, def);
8454 : }
8455 : /* Adjust the input so we pick up the partially reduced value
8456 : for the skip edge in vect_create_epilog_for_reduction. */
8457 4105 : accumulator->reduc_input = def;
8458 : /* And the reduction could be carried out using a different sign. */
8459 4105 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8460 235 : def = gimple_convert (&stmts, vectype_out, def);
8461 4105 : edge e;
8462 4105 : if ((e = loop_vinfo->main_loop_edge)
8463 4105 : || (e = loop_vinfo->skip_this_loop_edge))
8464 : {
8465 : /* While we'd like to insert on the edge this will split
8466 : blocks and disturb bookkeeping, we also will eventually
8467 : need this on the skip edge. Rely on sinking to
8468 : fixup optimal placement and insert in the pred. */
8469 3882 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
8470 : /* Insert before a cond that eventually skips the
8471 : epilogue. */
8472 3882 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
8473 3865 : gsi_prev (&gsi);
8474 3882 : gsi_insert_seq_after (&gsi, stmts, GSI_CONTINUE_LINKING);
8475 : }
8476 : else
8477 223 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop),
8478 : stmts);
8479 : }
8480 4108 : if (loop_vinfo->main_loop_edge)
8481 3885 : vec_initial_defs[0]
8482 3885 : = vect_get_main_loop_result (loop_vinfo, def,
8483 3885 : vec_initial_defs[0]);
8484 : else
8485 223 : vec_initial_defs.safe_push (def);
8486 : }
8487 :
8488 : /* Generate the reduction PHIs upfront. */
8489 47622 : for (i = 0; i < vec_num; i++)
8490 : {
8491 24760 : tree vec_init_def = vec_initial_defs[i];
8492 : /* Create the reduction-phi that defines the reduction
8493 : operand. */
8494 24760 : gphi *new_phi = create_phi_node (vec_dest, loop->header);
8495 24760 : add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
8496 : UNKNOWN_LOCATION);
8497 :
8498 : /* The loop-latch arg is set in epilogue processing. */
8499 :
8500 24760 : slp_node->push_vec_def (new_phi);
8501 : }
8502 :
8503 22862 : return true;
8504 22862 : }
8505 :
8506 : /* Vectorizes LC PHIs. */
8507 :
8508 : bool
8509 196356 : vectorizable_lc_phi (loop_vec_info loop_vinfo,
8510 : stmt_vec_info stmt_info,
8511 : slp_tree slp_node)
8512 : {
8513 196356 : if (!loop_vinfo
8514 196356 : || !is_a <gphi *> (stmt_info->stmt)
8515 235229 : || gimple_phi_num_args (stmt_info->stmt) != 1)
8516 : return false;
8517 :
8518 820 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8519 0 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
8520 : return false;
8521 :
8522 : /* Deal with copies from externs or constants that disguise as
8523 : loop-closed PHI nodes (PR97886). */
8524 820 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
8525 : SLP_TREE_VECTYPE (slp_node)))
8526 : {
8527 0 : if (dump_enabled_p ())
8528 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8529 : "incompatible vector types for invariants\n");
8530 : return false;
8531 : }
8532 :
8533 : /* ??? This can happen with data vs. mask uses of boolean. */
8534 820 : if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
8535 820 : SLP_TREE_VECTYPE
8536 : (SLP_TREE_CHILDREN (slp_node)[0])))
8537 : {
8538 0 : if (dump_enabled_p ())
8539 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8540 : "missed mask promotion\n");
8541 : return false;
8542 : }
8543 :
8544 820 : SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
8545 820 : return true;
8546 : }
8547 :
8548 : bool
8549 529 : vect_transform_lc_phi (loop_vec_info loop_vinfo,
8550 : stmt_vec_info stmt_info,
8551 : slp_tree slp_node)
8552 : {
8553 :
8554 529 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8555 529 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8556 529 : basic_block bb = gimple_bb (stmt_info->stmt);
8557 529 : edge e = single_pred_edge (bb);
8558 529 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8559 529 : auto_vec<tree> vec_oprnds;
8560 529 : vect_get_vec_defs (loop_vinfo, slp_node, true, &vec_oprnds);
8561 1702 : for (unsigned i = 0; i < vec_oprnds.length (); i++)
8562 : {
8563 : /* Create the vectorized LC PHI node. */
8564 644 : gphi *new_phi = create_phi_node (vec_dest, bb);
8565 644 : add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
8566 644 : slp_node->push_vec_def (new_phi);
8567 : }
8568 :
8569 529 : return true;
8570 529 : }
8571 :
8572 : /* Vectorizes PHIs. */
8573 :
8574 : bool
8575 156942 : vectorizable_phi (bb_vec_info vinfo,
8576 : stmt_vec_info stmt_info,
8577 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8578 : {
8579 156942 : if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
8580 : return false;
8581 :
8582 75454 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
8583 : return false;
8584 :
8585 75454 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8586 :
8587 75454 : if (cost_vec) /* transformation not required. */
8588 : {
8589 : slp_tree child;
8590 : unsigned i;
8591 205171 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), i, child)
8592 144469 : if (!child)
8593 : {
8594 0 : if (dump_enabled_p ())
8595 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8596 : "PHI node with unvectorized backedge def\n");
8597 : return false;
8598 : }
8599 144469 : else if (!vect_maybe_update_slp_op_vectype (child, vectype))
8600 : {
8601 26 : if (dump_enabled_p ())
8602 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8603 : "incompatible vector types for invariants\n");
8604 : return false;
8605 : }
8606 144443 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8607 144443 : && !useless_type_conversion_p (vectype,
8608 : SLP_TREE_VECTYPE (child)))
8609 : {
8610 : /* With bools we can have mask and non-mask precision vectors
8611 : or different non-mask precisions. while pattern recog is
8612 : supposed to guarantee consistency here bugs in it can cause
8613 : mismatches (PR103489 and PR103800 for example).
8614 : Deal with them here instead of ICEing later. */
8615 18 : if (dump_enabled_p ())
8616 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8617 : "incompatible vector type setup from "
8618 : "bool pattern detection\n");
8619 : return false;
8620 : }
8621 :
8622 : /* For single-argument PHIs assume coalescing which means zero cost
8623 : for the scalar and the vector PHIs. This avoids artificially
8624 : favoring the vector path (but may pessimize it in some cases). */
8625 60702 : if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
8626 54984 : record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, slp_node),
8627 : vector_stmt, slp_node, vectype, 0, vect_body);
8628 60702 : SLP_TREE_TYPE (slp_node) = phi_info_type;
8629 60702 : return true;
8630 : }
8631 :
8632 14708 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8633 14708 : basic_block bb = gimple_bb (stmt_info->stmt);
8634 14708 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8635 14708 : auto_vec<gphi *> new_phis;
8636 52851 : for (unsigned i = 0; i < gimple_phi_num_args (stmt_info->stmt); ++i)
8637 : {
8638 38143 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
8639 :
8640 : /* Skip not yet vectorized defs. */
8641 38631 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8642 38143 : && SLP_TREE_VEC_DEFS (child).is_empty ())
8643 488 : continue;
8644 :
8645 37655 : auto_vec<tree> vec_oprnds;
8646 37655 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[i], &vec_oprnds);
8647 37655 : if (!new_phis.exists ())
8648 : {
8649 14708 : new_phis.create (vec_oprnds.length ());
8650 45593 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8651 : {
8652 : /* Create the vectorized LC PHI node. */
8653 16177 : new_phis.quick_push (create_phi_node (vec_dest, bb));
8654 16177 : slp_node->push_vec_def (new_phis[j]);
8655 : }
8656 : }
8657 37655 : edge e = gimple_phi_arg_edge (as_a <gphi *> (stmt_info->stmt), i);
8658 80327 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8659 42672 : add_phi_arg (new_phis[j], vec_oprnds[j], e, UNKNOWN_LOCATION);
8660 37655 : }
8661 : /* We should have at least one already vectorized child. */
8662 14708 : gcc_assert (new_phis.exists ());
8663 :
8664 14708 : return true;
8665 14708 : }
8666 :
8667 : /* Vectorizes first order recurrences. An overview of the transformation
8668 : is described below. Suppose we have the following loop.
8669 :
8670 : int t = 0;
8671 : for (int i = 0; i < n; ++i)
8672 : {
8673 : b[i] = a[i] - t;
8674 : t = a[i];
8675 : }
8676 :
8677 : There is a first-order recurrence on 'a'. For this loop, the scalar IR
8678 : looks (simplified) like:
8679 :
8680 : scalar.preheader:
8681 : init = 0;
8682 :
8683 : scalar.body:
8684 : i = PHI <0(scalar.preheader), i+1(scalar.body)>
8685 : _2 = PHI <(init(scalar.preheader), <_1(scalar.body)>
8686 : _1 = a[i]
8687 : b[i] = _1 - _2
8688 : if (i < n) goto scalar.body
8689 :
8690 : In this example, _2 is a recurrence because it's value depends on the
8691 : previous iteration. We vectorize this as (VF = 4)
8692 :
8693 : vector.preheader:
8694 : vect_init = vect_cst(..., ..., ..., 0)
8695 :
8696 : vector.body
8697 : i = PHI <0(vector.preheader), i+4(vector.body)>
8698 : vect_1 = PHI <vect_init(vector.preheader), v2(vector.body)>
8699 : vect_2 = a[i, i+1, i+2, i+3];
8700 : vect_3 = vec_perm (vect_1, vect_2, { 3, 4, 5, 6 })
8701 : b[i, i+1, i+2, i+3] = vect_2 - vect_3
8702 : if (..) goto vector.body
8703 :
8704 : In this function, vectorizable_recurr, we code generate both the
8705 : vector PHI node and the permute since those together compute the
8706 : vectorized value of the scalar PHI. We do not yet have the
8707 : backedge value to fill in there nor into the vec_perm. Those
8708 : are filled in vect_schedule_scc.
8709 :
8710 : TODO: Since the scalar loop does not have a use of the recurrence
8711 : outside of the loop the natural way to implement peeling via
8712 : vectorizing the live value doesn't work. For now peeling of loops
8713 : with a recurrence is not implemented. For SLP the supported cases
8714 : are restricted to those requiring a single vector recurrence PHI. */
8715 :
8716 : bool
8717 195581 : vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
8718 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8719 : {
8720 195581 : if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
8721 : return false;
8722 :
8723 38098 : gphi *phi = as_a<gphi *> (stmt_info->stmt);
8724 :
8725 : /* So far we only support first-order recurrence auto-vectorization. */
8726 38098 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
8727 : return false;
8728 :
8729 418 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8730 418 : unsigned ncopies = vect_get_num_copies (loop_vinfo, slp_node);
8731 418 : poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8732 418 : unsigned dist = SLP_TREE_LANES (slp_node);
8733 : /* We need to be able to make progress with a single vector. */
8734 418 : if (maybe_gt (dist * 2, nunits))
8735 : {
8736 0 : if (dump_enabled_p ())
8737 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8738 : "first order recurrence exceeds half of "
8739 : "a vector\n");
8740 : return false;
8741 : }
8742 :
8743 : /* We need to be able to build a { ..., a, b } init vector with
8744 : dist number of distinct trailing values. Always possible
8745 : when dist == 1 or when nunits is constant or when the initializations
8746 : are uniform. */
8747 418 : tree uniform_initval = NULL_TREE;
8748 418 : edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
8749 1696 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8750 : {
8751 454 : gphi *phi = as_a <gphi *> (s->stmt);
8752 454 : if (! uniform_initval)
8753 418 : uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8754 36 : else if (! operand_equal_p (uniform_initval,
8755 36 : PHI_ARG_DEF_FROM_EDGE (phi, pe)))
8756 : {
8757 : uniform_initval = NULL_TREE;
8758 : break;
8759 : }
8760 : }
8761 418 : if (!uniform_initval && !nunits.is_constant ())
8762 : {
8763 : if (dump_enabled_p ())
8764 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8765 : "cannot build initialization vector for "
8766 : "first order recurrence\n");
8767 : return false;
8768 : }
8769 :
8770 : /* First-order recurrence autovectorization needs to handle permutation
8771 : with indices = [nunits-1, nunits, nunits+1, ...]. */
8772 418 : vec_perm_builder sel (nunits, 1, 3);
8773 1672 : for (int i = 0; i < 3; ++i)
8774 1254 : sel.quick_push (nunits - dist + i);
8775 418 : vec_perm_indices indices (sel, 2, nunits);
8776 :
8777 418 : if (cost_vec) /* transformation not required. */
8778 : {
8779 373 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
8780 : indices))
8781 : return false;
8782 :
8783 : /* We eventually need to set a vector type on invariant
8784 : arguments. */
8785 : unsigned j;
8786 : slp_tree child;
8787 783 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
8788 522 : if (!vect_maybe_update_slp_op_vectype (child, vectype))
8789 : {
8790 0 : if (dump_enabled_p ())
8791 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8792 : "incompatible vector types for "
8793 : "invariants\n");
8794 : return false;
8795 : }
8796 :
8797 : /* Verify we have set up compatible types. */
8798 261 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8799 261 : slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
8800 261 : tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
8801 261 : if (!types_compatible_p (latch_vectype, vectype))
8802 : return false;
8803 :
8804 : /* The recurrence costs the initialization vector and one permute
8805 : for each copy. With SLP the prologue value is explicitly
8806 : represented and costed separately. */
8807 261 : unsigned prologue_cost = 0;
8808 261 : unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
8809 : slp_node, 0, vect_body);
8810 261 : if (dump_enabled_p ())
8811 53 : dump_printf_loc (MSG_NOTE, vect_location,
8812 : "vectorizable_recurr: inside_cost = %d, "
8813 : "prologue_cost = %d .\n", inside_cost,
8814 : prologue_cost);
8815 :
8816 261 : SLP_TREE_TYPE (slp_node) = recurr_info_type;
8817 261 : return true;
8818 : }
8819 :
8820 45 : tree vec_init;
8821 45 : if (! uniform_initval)
8822 : {
8823 6 : vec<constructor_elt, va_gc> *v = NULL;
8824 6 : vec_alloc (v, nunits.to_constant ());
8825 39 : for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
8826 27 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
8827 : build_zero_cst (TREE_TYPE (vectype)));
8828 39 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8829 : {
8830 21 : gphi *phi = as_a <gphi *> (s->stmt);
8831 21 : tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8832 21 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
8833 21 : TREE_TYPE (preheader)))
8834 : {
8835 0 : gimple_seq stmts = NULL;
8836 0 : preheader = gimple_convert (&stmts,
8837 0 : TREE_TYPE (vectype), preheader);
8838 0 : gsi_insert_seq_on_edge_immediate (pe, stmts);
8839 : }
8840 21 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
8841 : }
8842 6 : vec_init = build_constructor (vectype, v);
8843 : }
8844 : else
8845 : vec_init = uniform_initval;
8846 45 : vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
8847 :
8848 : /* Create the vectorized first-order PHI node. */
8849 45 : tree vec_dest = vect_get_new_vect_var (vectype,
8850 : vect_simple_var, "vec_recur_");
8851 45 : basic_block bb = gimple_bb (phi);
8852 45 : gphi *new_phi = create_phi_node (vec_dest, bb);
8853 45 : add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
8854 :
8855 : /* Insert shuffles the first-order recurrence autovectorization.
8856 : result = VEC_PERM <vec_recur, vect_1, index[nunits-1, nunits, ...]>. */
8857 45 : tree perm = vect_gen_perm_mask_checked (vectype, indices);
8858 :
8859 : /* Insert the required permute after the latch definition. The
8860 : second and later operands are tentative and will be updated when we have
8861 : vectorized the latch definition. */
8862 45 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8863 45 : gimple *latch_def = SSA_NAME_DEF_STMT (PHI_ARG_DEF_FROM_EDGE (phi, le));
8864 45 : gimple_stmt_iterator gsi2 = gsi_for_stmt (latch_def);
8865 51 : do
8866 : {
8867 51 : gsi_next (&gsi2);
8868 : }
8869 : /* Skip inserted vectorized stmts for the latch definition. We have to
8870 : insert after those. */
8871 96 : while (gsi_stmt (gsi2) && gimple_uid (gsi_stmt (gsi2)) == 0);
8872 :
8873 127 : for (unsigned i = 0; i < ncopies; ++i)
8874 : {
8875 82 : vec_dest = make_ssa_name (vectype);
8876 82 : gassign *vperm
8877 127 : = gimple_build_assign (vec_dest, VEC_PERM_EXPR,
8878 45 : i == 0 ? gimple_phi_result (new_phi) : NULL,
8879 : NULL, perm);
8880 82 : vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
8881 :
8882 82 : slp_node->push_vec_def (vperm);
8883 : }
8884 :
8885 : return true;
8886 418 : }
8887 :
8888 : /* Return true if VECTYPE represents a vector that requires lowering
8889 : by the vector lowering pass. */
8890 :
8891 : bool
8892 668881 : vect_emulated_vector_p (tree vectype)
8893 : {
8894 1337762 : return (!VECTOR_MODE_P (TYPE_MODE (vectype))
8895 673032 : && (!VECTOR_BOOLEAN_TYPE_P (vectype)
8896 4109 : || TYPE_PRECISION (TREE_TYPE (vectype)) != 1));
8897 : }
8898 :
8899 : /* Return true if we can emulate CODE on an integer mode representation
8900 : of a vector. */
8901 :
8902 : bool
8903 12287 : vect_can_vectorize_without_simd_p (tree_code code)
8904 : {
8905 12287 : switch (code)
8906 : {
8907 : case PLUS_EXPR:
8908 : case MINUS_EXPR:
8909 : case NEGATE_EXPR:
8910 : case BIT_AND_EXPR:
8911 : case BIT_IOR_EXPR:
8912 : case BIT_XOR_EXPR:
8913 : case BIT_NOT_EXPR:
8914 : return true;
8915 :
8916 11670 : default:
8917 11670 : return false;
8918 : }
8919 : }
8920 :
8921 : /* Likewise, but taking a code_helper. */
8922 :
8923 : bool
8924 1004 : vect_can_vectorize_without_simd_p (code_helper code)
8925 : {
8926 1004 : return (code.is_tree_code ()
8927 1004 : && vect_can_vectorize_without_simd_p (tree_code (code)));
8928 : }
8929 :
8930 : /* Create vector init for vectorized iv. */
8931 : static tree
8932 919 : vect_create_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
8933 : tree step_expr, poly_uint64 nunits,
8934 : tree vectype,
8935 : enum vect_induction_op_type induction_type)
8936 : {
8937 919 : unsigned HOST_WIDE_INT const_nunits;
8938 919 : tree vec_shift, vec_init, new_name;
8939 919 : unsigned i;
8940 919 : tree itype = TREE_TYPE (vectype);
8941 :
8942 : /* iv_loop is the loop to be vectorized. Create:
8943 : vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr). */
8944 919 : new_name = gimple_convert (stmts, itype, init_expr);
8945 919 : switch (induction_type)
8946 : {
8947 18 : case vect_step_op_shr:
8948 18 : case vect_step_op_shl:
8949 : /* Build the Initial value from shift_expr. */
8950 18 : vec_init = gimple_build_vector_from_val (stmts,
8951 : vectype,
8952 : new_name);
8953 18 : vec_shift = gimple_build (stmts, VEC_SERIES_EXPR, vectype,
8954 : build_zero_cst (itype), step_expr);
8955 18 : vec_init = gimple_build (stmts,
8956 : (induction_type == vect_step_op_shr
8957 : ? RSHIFT_EXPR : LSHIFT_EXPR),
8958 : vectype, vec_init, vec_shift);
8959 18 : break;
8960 :
8961 825 : case vect_step_op_neg:
8962 825 : {
8963 825 : vec_init = gimple_build_vector_from_val (stmts,
8964 : vectype,
8965 : new_name);
8966 825 : tree vec_neg = gimple_build (stmts, NEGATE_EXPR,
8967 : vectype, vec_init);
8968 : /* The encoding has 2 interleaved stepped patterns. */
8969 825 : vec_perm_builder sel (nunits, 2, 3);
8970 825 : sel.quick_grow (6);
8971 4125 : for (i = 0; i < 3; i++)
8972 : {
8973 2475 : sel[2 * i] = i;
8974 2475 : sel[2 * i + 1] = i + nunits;
8975 : }
8976 825 : vec_perm_indices indices (sel, 2, nunits);
8977 : /* Don't use vect_gen_perm_mask_checked since can_vec_perm_const_p may
8978 : fail when vec_init is const vector. In that situation vec_perm is not
8979 : really needed. */
8980 825 : tree perm_mask_even
8981 825 : = vect_gen_perm_mask_any (vectype, indices);
8982 825 : vec_init = gimple_build (stmts, VEC_PERM_EXPR,
8983 : vectype,
8984 : vec_init, vec_neg,
8985 : perm_mask_even);
8986 825 : }
8987 825 : break;
8988 :
8989 76 : case vect_step_op_mul:
8990 76 : {
8991 : /* Use unsigned mult to avoid UD integer overflow. */
8992 76 : gcc_assert (nunits.is_constant (&const_nunits));
8993 76 : tree utype = unsigned_type_for (itype);
8994 76 : tree uvectype = build_vector_type (utype,
8995 76 : TYPE_VECTOR_SUBPARTS (vectype));
8996 76 : new_name = gimple_convert (stmts, utype, new_name);
8997 76 : vec_init = gimple_build_vector_from_val (stmts,
8998 : uvectype,
8999 : new_name);
9000 76 : tree_vector_builder elts (uvectype, const_nunits, 1);
9001 76 : tree elt_step = build_one_cst (utype);
9002 :
9003 76 : elts.quick_push (elt_step);
9004 660 : for (i = 1; i < const_nunits; i++)
9005 : {
9006 : /* Create: new_name_i = new_name + step_expr. */
9007 508 : elt_step = gimple_build (stmts, MULT_EXPR,
9008 : utype, elt_step, step_expr);
9009 508 : elts.quick_push (elt_step);
9010 : }
9011 : /* Create a vector from [new_name_0, new_name_1, ...,
9012 : new_name_nunits-1]. */
9013 76 : tree vec_mul = gimple_build_vector (stmts, &elts);
9014 76 : vec_init = gimple_build (stmts, MULT_EXPR, uvectype,
9015 : vec_init, vec_mul);
9016 76 : vec_init = gimple_convert (stmts, vectype, vec_init);
9017 76 : }
9018 76 : break;
9019 :
9020 0 : default:
9021 0 : gcc_unreachable ();
9022 : }
9023 :
9024 919 : return vec_init;
9025 : }
9026 :
9027 : /* Peel init_expr by skip_niter for induction_type. */
9028 : tree
9029 84 : vect_peel_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
9030 : tree skip_niters, tree step_expr,
9031 : enum vect_induction_op_type induction_type,
9032 : bool early_exit_p)
9033 : {
9034 84 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST || early_exit_p);
9035 84 : tree type = TREE_TYPE (init_expr);
9036 84 : unsigned prec = TYPE_PRECISION (type);
9037 84 : switch (induction_type)
9038 : {
9039 : /* neg inductions are typically not used for loop termination conditions but
9040 : are typically implemented as b = -b. That is every scalar iteration b is
9041 : negated. That means that for the initial value of b we will have to
9042 : determine whether the number of skipped iteration is a multiple of 2
9043 : because every 2 scalar iterations we are back at "b". */
9044 0 : case vect_step_op_neg:
9045 : /* For early exits the neg induction will always be the same value at the
9046 : start of the iteration. */
9047 0 : if (early_exit_p)
9048 : break;
9049 :
9050 0 : if (TREE_INT_CST_LOW (skip_niters) % 2)
9051 0 : init_expr = gimple_build (stmts, NEGATE_EXPR, type, init_expr);
9052 : /* else no change. */
9053 : break;
9054 :
9055 12 : case vect_step_op_shr:
9056 12 : case vect_step_op_shl:
9057 12 : skip_niters = fold_build1 (NOP_EXPR, type, skip_niters);
9058 12 : step_expr = fold_build1 (NOP_EXPR, type, step_expr);
9059 12 : step_expr = fold_build2 (MULT_EXPR, type, step_expr, skip_niters);
9060 : /* When shift mount >= precision, need to avoid UD.
9061 : In the original loop, there's no UD, and according to semantic,
9062 : init_expr should be 0 for lshr, ashl, and >>= (prec - 1) for ashr. */
9063 12 : if ((!tree_fits_uhwi_p (step_expr)
9064 12 : || tree_to_uhwi (step_expr) >= prec)
9065 6 : && !early_exit_p)
9066 : {
9067 6 : if (induction_type == vect_step_op_shl
9068 6 : || TYPE_UNSIGNED (type))
9069 4 : init_expr = build_zero_cst (type);
9070 : else
9071 2 : init_expr = gimple_build (stmts, RSHIFT_EXPR, type,
9072 : init_expr,
9073 4 : wide_int_to_tree (type, prec - 1));
9074 : }
9075 : else
9076 : {
9077 8 : init_expr = fold_build2 ((induction_type == vect_step_op_shr
9078 : ? RSHIFT_EXPR : LSHIFT_EXPR),
9079 : type, init_expr, step_expr);
9080 6 : init_expr = force_gimple_operand (init_expr, stmts, false, NULL);
9081 : }
9082 : break;
9083 :
9084 72 : case vect_step_op_mul:
9085 72 : {
9086 : /* Due to UB we can't support vect_step_op_mul with early break for now.
9087 : so assert and block. */
9088 72 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST);
9089 72 : tree utype = unsigned_type_for (type);
9090 72 : init_expr = gimple_convert (stmts, utype, init_expr);
9091 72 : wide_int skipn = wi::to_wide (skip_niters);
9092 72 : wide_int begin = wi::to_wide (step_expr);
9093 72 : auto_mpz base, exp, mod, res;
9094 72 : wi::to_mpz (begin, base, TYPE_SIGN (type));
9095 72 : wi::to_mpz (skipn, exp, UNSIGNED);
9096 72 : mpz_ui_pow_ui (mod, 2, TYPE_PRECISION (type));
9097 72 : mpz_powm (res, base, exp, mod);
9098 72 : begin = wi::from_mpz (utype, res, true);
9099 72 : tree mult_expr = wide_int_to_tree (utype, begin);
9100 72 : init_expr = gimple_build (stmts, MULT_EXPR, utype,
9101 : init_expr, mult_expr);
9102 72 : init_expr = gimple_convert (stmts, type, init_expr);
9103 72 : }
9104 72 : break;
9105 :
9106 0 : default:
9107 0 : gcc_unreachable ();
9108 : }
9109 :
9110 84 : return init_expr;
9111 : }
9112 :
9113 : /* Create vector step for vectorized iv. */
9114 : static tree
9115 1205 : vect_create_nonlinear_iv_step (gimple_seq* stmts, tree step_expr,
9116 : poly_uint64 vf,
9117 : enum vect_induction_op_type induction_type)
9118 : {
9119 1205 : tree expr = build_int_cst (TREE_TYPE (step_expr), vf);
9120 1205 : tree new_name = NULL;
9121 : /* Step should be pow (step, vf) for mult induction. */
9122 1205 : if (induction_type == vect_step_op_mul)
9123 : {
9124 76 : gcc_assert (vf.is_constant ());
9125 76 : wide_int begin = wi::to_wide (step_expr);
9126 :
9127 584 : for (unsigned i = 0; i != vf.to_constant () - 1; i++)
9128 508 : begin = wi::mul (begin, wi::to_wide (step_expr));
9129 :
9130 76 : new_name = wide_int_to_tree (TREE_TYPE (step_expr), begin);
9131 76 : }
9132 1129 : else if (induction_type == vect_step_op_neg)
9133 : /* Do nothing. */
9134 : ;
9135 : else
9136 18 : new_name = gimple_build (stmts, MULT_EXPR, TREE_TYPE (step_expr),
9137 : expr, step_expr);
9138 1205 : return new_name;
9139 : }
9140 :
9141 : static tree
9142 1205 : vect_create_nonlinear_iv_vec_step (loop_vec_info loop_vinfo,
9143 : stmt_vec_info stmt_info,
9144 : tree new_name, tree vectype,
9145 : enum vect_induction_op_type induction_type)
9146 : {
9147 : /* No step is needed for neg induction. */
9148 1205 : if (induction_type == vect_step_op_neg)
9149 : return NULL;
9150 :
9151 94 : tree t = unshare_expr (new_name);
9152 94 : gcc_assert (CONSTANT_CLASS_P (new_name)
9153 : || TREE_CODE (new_name) == SSA_NAME);
9154 94 : tree new_vec = build_vector_from_val (vectype, t);
9155 94 : tree vec_step = vect_init_vector (loop_vinfo, stmt_info,
9156 : new_vec, vectype, NULL);
9157 94 : return vec_step;
9158 : }
9159 :
9160 : /* Update vectorized iv with vect_step, induc_def is init. */
9161 : static tree
9162 1393 : vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
9163 : tree induc_def, tree vec_step,
9164 : enum vect_induction_op_type induction_type)
9165 : {
9166 1393 : tree vec_def = induc_def;
9167 1393 : switch (induction_type)
9168 : {
9169 76 : case vect_step_op_mul:
9170 76 : {
9171 : /* Use unsigned mult to avoid UD integer overflow. */
9172 76 : tree uvectype = unsigned_type_for (vectype);
9173 76 : vec_def = gimple_convert (stmts, uvectype, vec_def);
9174 76 : vec_step = gimple_convert (stmts, uvectype, vec_step);
9175 76 : vec_def = gimple_build (stmts, MULT_EXPR, uvectype,
9176 : vec_def, vec_step);
9177 76 : vec_def = gimple_convert (stmts, vectype, vec_def);
9178 : }
9179 76 : break;
9180 :
9181 12 : case vect_step_op_shr:
9182 12 : vec_def = gimple_build (stmts, RSHIFT_EXPR, vectype,
9183 : vec_def, vec_step);
9184 12 : break;
9185 :
9186 6 : case vect_step_op_shl:
9187 6 : vec_def = gimple_build (stmts, LSHIFT_EXPR, vectype,
9188 : vec_def, vec_step);
9189 6 : break;
9190 : case vect_step_op_neg:
9191 : vec_def = induc_def;
9192 : /* Do nothing. */
9193 : break;
9194 0 : default:
9195 0 : gcc_unreachable ();
9196 : }
9197 :
9198 1393 : return vec_def;
9199 :
9200 : }
9201 :
9202 : /* Function vectorizable_nonlinear_induction
9203 :
9204 : Check if STMT_INFO performs an nonlinear induction computation that can be
9205 : vectorized. If VEC_STMT is also passed, vectorize the induction PHI: create
9206 : a vectorized phi to replace it, put it in VEC_STMT, and add it to the same
9207 : basic block.
9208 : Return true if STMT_INFO is vectorizable in this way. */
9209 :
9210 : static bool
9211 9587 : vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
9212 : stmt_vec_info stmt_info,
9213 : slp_tree slp_node,
9214 : stmt_vector_for_cost *cost_vec)
9215 : {
9216 9587 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9217 9587 : unsigned ncopies;
9218 9587 : bool nested_in_vect_loop = false;
9219 9587 : class loop *iv_loop;
9220 9587 : tree vec_def;
9221 9587 : edge pe = loop_preheader_edge (loop);
9222 9587 : basic_block new_bb;
9223 9587 : tree vec_init, vec_step;
9224 9587 : tree new_name;
9225 9587 : gimple *new_stmt;
9226 9587 : gphi *induction_phi;
9227 9587 : tree induc_def, vec_dest;
9228 9587 : tree init_expr, step_expr;
9229 9587 : tree niters_skip;
9230 9587 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9231 9587 : unsigned i;
9232 9587 : gimple_stmt_iterator si;
9233 :
9234 9587 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9235 :
9236 9587 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9237 9587 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9238 9587 : enum vect_induction_op_type induction_type
9239 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9240 :
9241 9587 : gcc_assert (induction_type > vect_step_op_add);
9242 :
9243 9587 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
9244 9587 : gcc_assert (ncopies >= 1);
9245 :
9246 : /* FORNOW. Only handle nonlinear induction in the same loop. */
9247 9587 : if (nested_in_vect_loop_p (loop, stmt_info))
9248 : {
9249 0 : if (dump_enabled_p ())
9250 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9251 : "nonlinear induction in nested loop.\n");
9252 : return false;
9253 : }
9254 :
9255 9587 : iv_loop = loop;
9256 9587 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9257 :
9258 : /* TODO: Support multi-lane SLP for nonlinear iv. There should be separate
9259 : vector iv update for each iv and a permutation to generate wanted
9260 : vector iv. */
9261 9587 : if (SLP_TREE_LANES (slp_node) > 1)
9262 : {
9263 0 : if (dump_enabled_p ())
9264 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9265 : "SLP induction not supported for nonlinear"
9266 : " induction.\n");
9267 : return false;
9268 : }
9269 :
9270 9587 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
9271 : {
9272 0 : if (dump_enabled_p ())
9273 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9274 : "floating point nonlinear induction vectorization"
9275 : " not supported.\n");
9276 : return false;
9277 : }
9278 :
9279 9587 : step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9280 9587 : init_expr = vect_phi_initial_value (phi);
9281 9587 : gcc_assert (step_expr != NULL_TREE && init_expr != NULL
9282 : && TREE_CODE (step_expr) == INTEGER_CST);
9283 : /* step_expr should be aligned with init_expr,
9284 : .i.e. uint64 a >> 1, step is int, but vector<uint64> shift is used. */
9285 9587 : step_expr = fold_convert (TREE_TYPE (vectype), step_expr);
9286 :
9287 9587 : if (TREE_CODE (init_expr) == INTEGER_CST)
9288 4108 : init_expr = fold_convert (TREE_TYPE (vectype), init_expr);
9289 5479 : else if (!tree_nop_conversion_p (TREE_TYPE (vectype), TREE_TYPE (init_expr)))
9290 : {
9291 : /* INIT_EXPR could be a bit_field, bail out for such case. */
9292 4 : if (dump_enabled_p ())
9293 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9294 : "nonlinear induction vectorization failed:"
9295 : " component type of vectype is not a nop conversion"
9296 : " from type of init_expr.\n");
9297 : return false;
9298 : }
9299 :
9300 9583 : switch (induction_type)
9301 : {
9302 3729 : case vect_step_op_neg:
9303 3729 : if (maybe_eq (TYPE_VECTOR_SUBPARTS (vectype), 1u))
9304 : return false;
9305 3567 : if (TREE_CODE (init_expr) != INTEGER_CST
9306 282 : && TREE_CODE (init_expr) != REAL_CST)
9307 : {
9308 : /* Check for backend support of NEGATE_EXPR and vec_perm. */
9309 282 : if (!directly_supported_p (NEGATE_EXPR, vectype))
9310 0 : return false;
9311 :
9312 : /* The encoding has 2 interleaved stepped patterns. */
9313 282 : vec_perm_builder sel (nunits, 2, 3);
9314 282 : machine_mode mode = TYPE_MODE (vectype);
9315 282 : sel.quick_grow (6);
9316 1410 : for (i = 0; i < 3; i++)
9317 : {
9318 846 : sel[i * 2] = i;
9319 846 : sel[i * 2 + 1] = i + nunits;
9320 : }
9321 282 : vec_perm_indices indices (sel, 2, nunits);
9322 282 : if (!can_vec_perm_const_p (mode, mode, indices))
9323 0 : return false;
9324 282 : }
9325 : break;
9326 :
9327 1066 : case vect_step_op_mul:
9328 1066 : {
9329 : /* Check for backend support of MULT_EXPR. */
9330 1066 : if (!directly_supported_p (MULT_EXPR, vectype))
9331 : return false;
9332 :
9333 : /* ?? How to construct vector step for variable number vector.
9334 : [ 1, step, pow (step, 2), pow (step, 4), .. ]. */
9335 : if (!vf.is_constant ())
9336 : return false;
9337 : }
9338 : break;
9339 :
9340 4456 : case vect_step_op_shr:
9341 : /* Check for backend support of RSHIFT_EXPR. */
9342 4456 : if (!directly_supported_p (RSHIFT_EXPR, vectype, optab_vector))
9343 : return false;
9344 :
9345 : /* Don't shift more than type precision to avoid UD. */
9346 26 : if (!tree_fits_uhwi_p (step_expr)
9347 26 : || maybe_ge (nunits * tree_to_uhwi (step_expr),
9348 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9349 : return false;
9350 : break;
9351 :
9352 332 : case vect_step_op_shl:
9353 : /* Check for backend support of RSHIFT_EXPR. */
9354 332 : if (!directly_supported_p (LSHIFT_EXPR, vectype, optab_vector))
9355 : return false;
9356 :
9357 : /* Don't shift more than type precision to avoid UD. */
9358 12 : if (!tree_fits_uhwi_p (step_expr)
9359 12 : || maybe_ge (nunits * tree_to_uhwi (step_expr),
9360 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9361 : return false;
9362 :
9363 : break;
9364 :
9365 0 : default:
9366 0 : gcc_unreachable ();
9367 : }
9368 :
9369 4435 : if (cost_vec) /* transformation not required. */
9370 : {
9371 3516 : unsigned inside_cost = 0, prologue_cost = 0;
9372 : /* loop cost for vec_loop. Neg induction doesn't have any
9373 : inside_cost. */
9374 3516 : inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
9375 : slp_node, 0, vect_body);
9376 :
9377 : /* loop cost for vec_loop. Neg induction doesn't have any
9378 : inside_cost. */
9379 3516 : if (induction_type == vect_step_op_neg)
9380 2742 : inside_cost = 0;
9381 :
9382 : /* prologue cost for vec_init and vec_step. */
9383 3516 : prologue_cost = record_stmt_cost (cost_vec, 2, scalar_to_vec,
9384 : slp_node, 0, vect_prologue);
9385 :
9386 3516 : if (dump_enabled_p ())
9387 68 : dump_printf_loc (MSG_NOTE, vect_location,
9388 : "vect_model_induction_cost: inside_cost = %d, "
9389 : "prologue_cost = %d. \n", inside_cost,
9390 : prologue_cost);
9391 :
9392 3516 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9393 3516 : DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
9394 3516 : return true;
9395 : }
9396 :
9397 : /* Transform. */
9398 :
9399 : /* Compute a vector variable, initialized with the first VF values of
9400 : the induction variable. E.g., for an iv with IV_PHI='X' and
9401 : evolution S, for a vector of 4 units, we want to compute:
9402 : [X, X + S, X + 2*S, X + 3*S]. */
9403 :
9404 919 : if (dump_enabled_p ())
9405 32 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9406 :
9407 919 : pe = loop_preheader_edge (iv_loop);
9408 : /* Find the first insertion point in the BB. */
9409 919 : basic_block bb = gimple_bb (phi);
9410 919 : si = gsi_after_labels (bb);
9411 :
9412 919 : gimple_seq stmts = NULL;
9413 :
9414 919 : niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
9415 : /* If we are using the loop mask to "peel" for alignment then we need
9416 : to adjust the start value here. */
9417 919 : if (niters_skip != NULL_TREE)
9418 0 : init_expr = vect_peel_nonlinear_iv_init (&stmts, init_expr, niters_skip,
9419 : step_expr, induction_type, false);
9420 :
9421 919 : vec_init = vect_create_nonlinear_iv_init (&stmts, init_expr,
9422 : step_expr, nunits, vectype,
9423 : induction_type);
9424 919 : if (stmts)
9425 : {
9426 162 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9427 162 : gcc_assert (!new_bb);
9428 : }
9429 :
9430 919 : stmts = NULL;
9431 919 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9432 : vf, induction_type);
9433 919 : if (stmts)
9434 : {
9435 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9436 0 : gcc_assert (!new_bb);
9437 : }
9438 :
9439 919 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9440 : new_name, vectype,
9441 : induction_type);
9442 : /* Create the following def-use cycle:
9443 : loop prolog:
9444 : vec_init = ...
9445 : vec_step = ...
9446 : loop:
9447 : vec_iv = PHI <vec_init, vec_loop>
9448 : ...
9449 : STMT
9450 : ...
9451 : vec_loop = vec_iv + vec_step; */
9452 :
9453 : /* Create the induction-phi that defines the induction-operand. */
9454 919 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
9455 919 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
9456 919 : induc_def = PHI_RESULT (induction_phi);
9457 :
9458 : /* Create the iv update inside the loop. */
9459 919 : stmts = NULL;
9460 919 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9461 : induc_def, vec_step,
9462 : induction_type);
9463 :
9464 919 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9465 919 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9466 :
9467 : /* Set the arguments of the phi node: */
9468 919 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
9469 919 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
9470 : UNKNOWN_LOCATION);
9471 :
9472 919 : slp_node->push_vec_def (induction_phi);
9473 :
9474 : /* In case that vectorization factor (VF) is bigger than the number
9475 : of elements that we can fit in a vectype (nunits), we have to generate
9476 : more than one vector stmt - i.e - we need to "unroll" the
9477 : vector stmt by a factor VF/nunits. For more details see documentation
9478 : in vectorizable_operation. */
9479 :
9480 919 : if (ncopies > 1)
9481 : {
9482 286 : stmts = NULL;
9483 : /* FORNOW. This restriction should be relaxed. */
9484 286 : gcc_assert (!nested_in_vect_loop);
9485 :
9486 286 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9487 : nunits, induction_type);
9488 :
9489 286 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9490 : new_name, vectype,
9491 : induction_type);
9492 286 : vec_def = induc_def;
9493 1046 : for (i = 1; i < ncopies; i++)
9494 : {
9495 : /* vec_i = vec_prev + vec_step. */
9496 474 : stmts = NULL;
9497 474 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9498 : vec_def, vec_step,
9499 : induction_type);
9500 474 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9501 474 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9502 474 : slp_node->push_vec_def (new_stmt);
9503 : }
9504 : }
9505 :
9506 919 : if (dump_enabled_p ())
9507 64 : dump_printf_loc (MSG_NOTE, vect_location,
9508 : "transform induction: created def-use cycle: %G%G",
9509 32 : (gimple *) induction_phi, SSA_NAME_DEF_STMT (vec_def));
9510 :
9511 : return true;
9512 : }
9513 :
9514 : /* Return true if the scalar initial values and steps of the SLP induction
9515 : lanes allow the first CANDIDATE_NIVS IVs to be reused circularly for the
9516 : remaining lanes. */
9517 : static bool
9518 26 : vect_slp_induction_reuse_p (tree *steps, tree *inits, unsigned group_size,
9519 : unsigned HOST_WIDE_INT const_nunits,
9520 : unsigned candidate_nivs, unsigned nivs)
9521 : {
9522 26 : gcc_assert (candidate_nivs > 0);
9523 26 : gcc_assert (candidate_nivs < nivs);
9524 :
9525 : /* This function compares only STEPS and INITS, so all checked lanes must
9526 : precede the first wrap of the SLP group. */
9527 26 : gcc_assert (nivs * const_nunits <= group_size);
9528 :
9529 62 : for (unsigned ivn = candidate_nivs; ivn < nivs; ++ivn)
9530 : {
9531 42 : unsigned reuse_ivn = ivn % candidate_nivs;
9532 172 : for (unsigned HOST_WIDE_INT eltn = 0; eltn < const_nunits; ++eltn)
9533 : {
9534 136 : unsigned HOST_WIDE_INT elt = ivn * const_nunits + eltn;
9535 136 : unsigned HOST_WIDE_INT reused_elt
9536 136 : = reuse_ivn * const_nunits + eltn;
9537 :
9538 136 : if (!operand_equal_p (steps[elt], steps[reused_elt], 0)
9539 136 : || !operand_equal_p (inits[elt], inits[reused_elt], 0))
9540 : return false;
9541 : }
9542 : }
9543 :
9544 : return true;
9545 : }
9546 :
9547 : /* Function vectorizable_induction
9548 :
9549 : Check if STMT_INFO performs an induction computation that can be vectorized.
9550 : If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
9551 : phi to replace it, put it in VEC_STMT, and add it to the same basic block.
9552 : Return true if STMT_INFO is vectorizable in this way. */
9553 :
9554 : bool
9555 337067 : vectorizable_induction (loop_vec_info loop_vinfo,
9556 : stmt_vec_info stmt_info,
9557 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
9558 : {
9559 337067 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9560 337067 : bool nested_in_vect_loop = false;
9561 337067 : class loop *iv_loop;
9562 337067 : tree vec_def;
9563 337067 : edge pe = loop_preheader_edge (loop);
9564 337067 : basic_block new_bb;
9565 337067 : tree vec_init = NULL_TREE, vec_step, t;
9566 337067 : tree new_name;
9567 337067 : gphi *induction_phi;
9568 337067 : tree induc_def, vec_dest;
9569 337067 : unsigned i;
9570 337067 : tree index_vectype = NULL_TREE;
9571 337067 : gimple_stmt_iterator si;
9572 337067 : enum vect_induction_op_type induction_type
9573 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9574 :
9575 337067 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9576 179584 : if (!phi)
9577 : return false;
9578 :
9579 179584 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
9580 : return false;
9581 :
9582 : /* Make sure it was recognized as induction computation. */
9583 179584 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
9584 : return false;
9585 :
9586 : /* Handle nonlinear induction in a separate place. */
9587 175493 : if (induction_type != vect_step_op_add)
9588 9587 : return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
9589 9587 : slp_node, cost_vec);
9590 :
9591 165906 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9592 165906 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9593 :
9594 : /* FORNOW. These restrictions should be relaxed. */
9595 165906 : if (nested_in_vect_loop_p (loop, stmt_info))
9596 : {
9597 811 : imm_use_iterator imm_iter;
9598 811 : use_operand_p use_p;
9599 811 : gimple *exit_phi;
9600 811 : edge latch_e;
9601 811 : tree loop_arg;
9602 :
9603 811 : exit_phi = NULL;
9604 811 : latch_e = loop_latch_edge (loop->inner);
9605 811 : loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
9606 1660 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
9607 : {
9608 871 : gimple *use_stmt = USE_STMT (use_p);
9609 871 : if (is_gimple_debug (use_stmt))
9610 36 : continue;
9611 :
9612 835 : if (!flow_bb_inside_loop_p (loop->inner, gimple_bb (use_stmt)))
9613 : {
9614 : exit_phi = use_stmt;
9615 : break;
9616 : }
9617 811 : }
9618 811 : if (exit_phi)
9619 : {
9620 22 : stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi);
9621 22 : if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
9622 6 : && !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
9623 : {
9624 16 : if (dump_enabled_p ())
9625 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9626 : "inner-loop induction only used outside "
9627 : "of the outer vectorized loop.\n");
9628 16 : return false;
9629 : }
9630 : }
9631 :
9632 795 : nested_in_vect_loop = true;
9633 795 : iv_loop = loop->inner;
9634 : }
9635 : else
9636 : iv_loop = loop;
9637 165890 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9638 :
9639 165890 : if (!nunits.is_constant () && SLP_TREE_LANES (slp_node) != 1)
9640 : {
9641 : /* The current SLP code creates the step value element-by-element. */
9642 : if (dump_enabled_p ())
9643 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9644 : "SLP induction not supported for variable-length"
9645 : " vectors.\n");
9646 : return false;
9647 : }
9648 :
9649 165890 : if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
9650 : {
9651 12 : if (dump_enabled_p ())
9652 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9653 : "floating point induction vectorization disabled\n");
9654 : return false;
9655 : }
9656 :
9657 165878 : tree step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9658 165878 : gcc_assert (step_expr != NULL_TREE);
9659 331732 : if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
9660 331631 : && !type_has_mode_precision_p (TREE_TYPE (step_expr)))
9661 : {
9662 12 : if (dump_enabled_p ())
9663 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9664 : "bit-precision induction vectorization not "
9665 : "supported.\n");
9666 : return false;
9667 : }
9668 165866 : tree stept = TREE_TYPE (step_expr);
9669 165866 : tree step_vectype = get_same_sized_vectype (stept, vectype);
9670 165866 : stept = TREE_TYPE (step_vectype);
9671 :
9672 : /* Check for target support of the vectorized arithmetic used here. */
9673 165866 : if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
9674 165866 : || !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default))
9675 : return false;
9676 136276 : if (!nunits.is_constant ()
9677 136276 : || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9678 : {
9679 0 : if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default))
9680 : return false;
9681 : /* FLOAT_EXPR when computing VEC_INIT for float inductions. */
9682 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9683 : {
9684 0 : tree index_type = build_nonstandard_integer_type
9685 0 : (GET_MODE_BITSIZE (SCALAR_TYPE_MODE (stept)), 1);
9686 :
9687 0 : index_vectype = build_vector_type (index_type, nunits);
9688 0 : if (!can_float_p (TYPE_MODE (step_vectype),
9689 0 : TYPE_MODE (index_vectype), 1))
9690 : return false;
9691 : }
9692 : }
9693 :
9694 136276 : unsigned nvects = vect_get_num_copies (loop_vinfo, slp_node);
9695 136276 : if (cost_vec) /* transformation not required. */
9696 : {
9697 362442 : unsigned inside_cost = 0, prologue_cost = 0;
9698 : /* We eventually need to set a vector type on invariant
9699 : arguments. */
9700 : unsigned j;
9701 : slp_tree child;
9702 362442 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
9703 241628 : if (!vect_maybe_update_slp_op_vectype
9704 241628 : (child, SLP_TREE_VECTYPE (slp_node)))
9705 : {
9706 0 : if (dump_enabled_p ())
9707 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9708 : "incompatible vector types for "
9709 : "invariants\n");
9710 : return false;
9711 : }
9712 : /* loop cost for vec_loop. */
9713 120814 : inside_cost = record_stmt_cost (cost_vec, nvects,
9714 : vector_stmt, slp_node, 0, vect_body);
9715 : /* prologue cost for vec_init (if not nested) and step. */
9716 120814 : prologue_cost = record_stmt_cost (cost_vec, 1 + !nested_in_vect_loop,
9717 : scalar_to_vec,
9718 : slp_node, 0, vect_prologue);
9719 120814 : if (dump_enabled_p ())
9720 4152 : dump_printf_loc (MSG_NOTE, vect_location,
9721 : "vect_model_induction_cost: inside_cost = %d, "
9722 : "prologue_cost = %d .\n", inside_cost,
9723 : prologue_cost);
9724 :
9725 120814 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9726 120814 : DUMP_VECT_SCOPE ("vectorizable_induction");
9727 120814 : return true;
9728 : }
9729 :
9730 : /* Transform. */
9731 :
9732 : /* Compute a vector variable, initialized with the first VF values of
9733 : the induction variable. E.g., for an iv with IV_PHI='X' and
9734 : evolution S, for a vector of 4 units, we want to compute:
9735 : [X, X + S, X + 2*S, X + 3*S]. */
9736 :
9737 15462 : if (dump_enabled_p ())
9738 2796 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9739 :
9740 15462 : pe = loop_preheader_edge (iv_loop);
9741 : /* Find the first insertion point in the BB. */
9742 15462 : basic_block bb = gimple_bb (phi);
9743 15462 : si = gsi_after_labels (bb);
9744 :
9745 : /* For SLP induction we have to generate several IVs as for example
9746 : with group size 3 we need
9747 : [i0, i1, i2, i0 + S0] [i1 + S1, i2 + S2, i0 + 2*S0, i1 + 2*S1]
9748 : [i2 + 2*S2, i0 + 3*S0, i1 + 3*S1, i2 + 3*S2]. */
9749 15462 : gimple_stmt_iterator incr_si;
9750 15462 : bool insert_after;
9751 15462 : standard_iv_increment_position (iv_loop, &incr_si, &insert_after);
9752 :
9753 : /* The initial values are vectorized, but any lanes > group_size
9754 : need adjustment. */
9755 15462 : slp_tree init_node
9756 15462 : = SLP_TREE_CHILDREN (slp_node)[pe->dest_idx];
9757 :
9758 : /* Gather steps. Since we do not vectorize inductions as
9759 : cycles we have to reconstruct the step from SCEV data. */
9760 15462 : unsigned group_size = SLP_TREE_LANES (slp_node);
9761 15462 : tree *steps = XALLOCAVEC (tree, group_size);
9762 15462 : tree *inits = XALLOCAVEC (tree, group_size);
9763 15462 : stmt_vec_info phi_info;
9764 47771 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, phi_info)
9765 : {
9766 16847 : steps[i] = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
9767 16847 : if (!init_node)
9768 16603 : inits[i] = gimple_phi_arg_def (as_a<gphi *> (phi_info->stmt),
9769 : pe->dest_idx);
9770 : }
9771 :
9772 : /* Now generate the IVs. */
9773 30924 : gcc_assert (multiple_p (nunits * nvects, group_size));
9774 15462 : unsigned nivs;
9775 15462 : unsigned HOST_WIDE_INT const_nunits;
9776 15462 : if (nested_in_vect_loop)
9777 : nivs = nvects;
9778 15239 : else if (nunits.is_constant (&const_nunits)
9779 15239 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9780 : {
9781 15239 : gcc_assert (!init_node);
9782 : /* Compute the number of distinct IVs we need. We can reduce the
9783 : number when later vector chunks are equal to earlier chunks. */
9784 15239 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
9785 15239 : unsigned group_sizep = group_size;
9786 15239 : if (group_sizep % const_nunits == 0)
9787 : {
9788 122 : group_sizep = group_sizep / const_nunits;
9789 122 : unsigned candidate_nivs
9790 122 : = least_common_multiple (group_sizep, const_nunits) / const_nunits;
9791 122 : if (candidate_nivs < nivs
9792 122 : && vect_slp_induction_reuse_p (steps, inits, group_size,
9793 : const_nunits, candidate_nivs, nivs))
9794 : {
9795 20 : if (dump_enabled_p ())
9796 16 : dump_printf_loc (MSG_NOTE, vect_location,
9797 : "reusing %u SLP induction IVs for %u "
9798 : "vector chunks\n",
9799 : candidate_nivs, nivs);
9800 : nivs = candidate_nivs;
9801 : }
9802 : }
9803 : }
9804 : else
9805 : {
9806 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
9807 : nivs = 1;
9808 : }
9809 15462 : gimple_seq init_stmts = NULL;
9810 15462 : gimple_seq lupdate_mul_stmts = NULL;
9811 15462 : tree lupdate_mul = NULL_TREE;
9812 15462 : if (!nested_in_vect_loop)
9813 : {
9814 15239 : if (nunits.is_constant (&const_nunits)
9815 15239 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9816 : {
9817 : /* The number of iterations covered in one vector iteration. */
9818 15239 : unsigned lup_mul = (nvects * const_nunits) / group_size;
9819 15239 : lupdate_mul
9820 15239 : = build_vector_from_val (step_vectype,
9821 15239 : SCALAR_FLOAT_TYPE_P (stept)
9822 28 : ? build_real_from_wide (stept, lup_mul,
9823 : UNSIGNED)
9824 30450 : : build_int_cstu (stept, lup_mul));
9825 : }
9826 : else
9827 : {
9828 0 : gimple_seq *update_stmts
9829 : = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
9830 : ? &init_stmts
9831 : : &lupdate_mul_stmts;
9832 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9833 : {
9834 0 : tree increment
9835 0 : = gimple_convert (update_stmts, integer_type_node,
9836 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9837 0 : lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept,
9838 : increment);
9839 : }
9840 : else
9841 0 : lupdate_mul = gimple_convert (update_stmts, stept,
9842 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9843 0 : lupdate_mul = gimple_build_vector_from_val (update_stmts,
9844 : step_vectype,
9845 : lupdate_mul);
9846 : }
9847 : }
9848 15462 : tree peel_mul = NULL_TREE;
9849 15462 : if (LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo))
9850 : {
9851 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9852 0 : peel_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept,
9853 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9854 : else
9855 0 : peel_mul = gimple_convert (&init_stmts, stept,
9856 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9857 0 : peel_mul = gimple_build_vector_from_val (&init_stmts,
9858 : step_vectype, peel_mul);
9859 : }
9860 15462 : tree step_mul = NULL_TREE;
9861 15462 : unsigned ivn;
9862 15462 : auto_vec<tree> vec_steps;
9863 31512 : for (ivn = 0; ivn < nivs; ++ivn)
9864 : {
9865 16050 : gimple_seq stmts = NULL;
9866 16050 : bool invariant = true;
9867 16050 : if (nunits.is_constant (&const_nunits)
9868 16050 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9869 : {
9870 16050 : tree_vector_builder step_elts (step_vectype, const_nunits, 1);
9871 16050 : tree_vector_builder init_elts (vectype, const_nunits, 1);
9872 16050 : tree_vector_builder mul_elts (step_vectype, const_nunits, 1);
9873 119364 : for (unsigned eltn = 0; eltn < const_nunits; ++eltn)
9874 : {
9875 : /* The scalar steps of the IVs. */
9876 87264 : tree elt = steps[(ivn*const_nunits + eltn) % group_size];
9877 87264 : elt = gimple_convert (&init_stmts, TREE_TYPE (step_vectype), elt);
9878 87264 : step_elts.quick_push (elt);
9879 87264 : if (!init_node)
9880 : {
9881 : /* The scalar inits of the IVs if not vectorized. */
9882 86006 : elt = inits[(ivn*const_nunits + eltn) % group_size];
9883 86006 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
9884 86006 : TREE_TYPE (elt)))
9885 260 : elt = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9886 260 : TREE_TYPE (vectype), elt);
9887 86006 : init_elts.quick_push (elt);
9888 : }
9889 : /* The number of steps to add to the initial values. */
9890 87264 : unsigned mul_elt = (ivn*const_nunits + eltn) / group_size;
9891 174528 : mul_elts.quick_push (SCALAR_FLOAT_TYPE_P (stept)
9892 174426 : ? build_real_from_wide (stept, mul_elt,
9893 : UNSIGNED)
9894 174426 : : build_int_cstu (stept, mul_elt));
9895 : }
9896 16050 : vec_step = gimple_build_vector (&init_stmts, &step_elts);
9897 16050 : step_mul = gimple_build_vector (&init_stmts, &mul_elts);
9898 16050 : if (!init_node)
9899 15793 : vec_init = gimple_build_vector (&init_stmts, &init_elts);
9900 16050 : }
9901 : else
9902 : {
9903 0 : tree step = gimple_convert (&init_stmts, stept, steps[0]);
9904 0 : if (init_node)
9905 : ;
9906 0 : else if (INTEGRAL_TYPE_P (stept))
9907 : {
9908 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
9909 : /* Build the initial value directly as a VEC_SERIES_EXPR. */
9910 0 : vec_init = gimple_build (&init_stmts, VEC_SERIES_EXPR,
9911 : step_vectype, new_name, step);
9912 0 : if (!useless_type_conversion_p (vectype, step_vectype))
9913 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9914 : vectype, vec_init);
9915 : }
9916 : else
9917 : {
9918 : /* Build:
9919 : [base, base, base, ...]
9920 : + (vectype) [0, 1, 2, ...] * [step, step, step, ...]. */
9921 0 : gcc_assert (SCALAR_FLOAT_TYPE_P (stept));
9922 0 : gcc_assert (flag_associative_math);
9923 0 : gcc_assert (index_vectype != NULL_TREE);
9924 :
9925 0 : tree index = build_index_vector (index_vectype, 0, 1);
9926 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
9927 0 : tree base_vec = gimple_build_vector_from_val (&init_stmts,
9928 : step_vectype,
9929 : new_name);
9930 0 : tree step_vec = gimple_build_vector_from_val (&init_stmts,
9931 : step_vectype,
9932 : step);
9933 0 : vec_init = gimple_build (&init_stmts, FLOAT_EXPR,
9934 : step_vectype, index);
9935 0 : vec_init = gimple_build (&init_stmts, MULT_EXPR,
9936 : step_vectype, vec_init, step_vec);
9937 0 : vec_init = gimple_build (&init_stmts, PLUS_EXPR,
9938 : step_vectype, vec_init, base_vec);
9939 0 : if (!useless_type_conversion_p (vectype, step_vectype))
9940 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9941 : vectype, vec_init);
9942 : }
9943 : /* iv_loop is nested in the loop to be vectorized. Generate:
9944 : vec_step = [S, S, S, S] */
9945 0 : t = unshare_expr (step);
9946 0 : gcc_assert (CONSTANT_CLASS_P (t)
9947 : || TREE_CODE (t) == SSA_NAME);
9948 0 : vec_step = gimple_build_vector_from_val (&init_stmts,
9949 : step_vectype, t);
9950 : }
9951 16050 : vec_steps.safe_push (vec_step);
9952 16050 : if (peel_mul)
9953 : {
9954 0 : if (!step_mul)
9955 : {
9956 0 : gcc_assert (!nunits.is_constant ());
9957 : step_mul = gimple_build (&init_stmts,
9958 : MINUS_EXPR, step_vectype,
9959 : build_zero_cst (step_vectype), peel_mul);
9960 : }
9961 : else
9962 0 : step_mul = gimple_build (&init_stmts,
9963 : MINUS_EXPR, step_vectype,
9964 : step_mul, peel_mul);
9965 : }
9966 :
9967 : /* Create the induction-phi that defines the induction-operand. */
9968 16050 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var,
9969 : "vec_iv_");
9970 16050 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
9971 16050 : induc_def = PHI_RESULT (induction_phi);
9972 :
9973 : /* Create the iv update inside the loop */
9974 16050 : tree up = vec_step;
9975 16050 : if (lupdate_mul)
9976 : {
9977 15793 : if (lupdate_mul_stmts)
9978 0 : gimple_seq_add_seq (&stmts, lupdate_mul_stmts);
9979 15793 : up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step,
9980 : lupdate_mul);
9981 : }
9982 16050 : vec_def = gimple_convert (&stmts, step_vectype, induc_def);
9983 16050 : vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
9984 16050 : vec_def = gimple_convert (&stmts, vectype, vec_def);
9985 16050 : insert_iv_increment (&incr_si, insert_after, stmts);
9986 16050 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
9987 : UNKNOWN_LOCATION);
9988 :
9989 16050 : if (init_node)
9990 257 : vec_init = vect_get_slp_vect_def (init_node, ivn);
9991 16050 : if (!nested_in_vect_loop
9992 16050 : && step_mul
9993 16050 : && !integer_zerop (step_mul))
9994 : {
9995 15325 : gcc_assert (invariant);
9996 15325 : vec_def = gimple_convert (&init_stmts, step_vectype, vec_init);
9997 15325 : up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
9998 : vec_step, step_mul);
9999 15325 : vec_def = gimple_build (&init_stmts, PLUS_EXPR, step_vectype,
10000 : vec_def, up);
10001 15325 : vec_init = gimple_convert (&init_stmts, vectype, vec_def);
10002 : }
10003 :
10004 : /* Set the arguments of the phi node: */
10005 16050 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
10006 :
10007 16050 : slp_node->push_vec_def (induction_phi);
10008 : }
10009 15462 : if (!nested_in_vect_loop)
10010 : {
10011 : /* Fill up to the number of vectors we need for the whole group. */
10012 15239 : if (nunits.is_constant (&const_nunits)
10013 15239 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10014 15239 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
10015 : else
10016 : nivs = 1;
10017 15239 : vec_steps.reserve (nivs-ivn);
10018 15239 : unsigned generated_nivs = ivn;
10019 15239 : gcc_assert (generated_nivs > 0);
10020 15275 : for (; ivn < nivs; ++ivn)
10021 : {
10022 36 : unsigned reuse_ivn = ivn % generated_nivs;
10023 36 : slp_node->push_vec_def (SLP_TREE_VEC_DEFS (slp_node)[reuse_ivn]);
10024 36 : vec_steps.quick_push (vec_steps[reuse_ivn]);
10025 : }
10026 : }
10027 :
10028 : /* Re-use IVs when we can. We are generating further vector
10029 : stmts by adding VF' * stride to the IVs generated above. */
10030 15462 : if (ivn < nvects)
10031 : {
10032 3406 : if (nunits.is_constant (&const_nunits)
10033 3406 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10034 : {
10035 3406 : unsigned vfp = (least_common_multiple (group_size, const_nunits)
10036 3406 : / group_size);
10037 3406 : lupdate_mul
10038 3406 : = build_vector_from_val (step_vectype,
10039 3406 : SCALAR_FLOAT_TYPE_P (stept)
10040 8 : ? build_real_from_wide (stept,
10041 8 : vfp, UNSIGNED)
10042 6804 : : build_int_cstu (stept, vfp));
10043 : }
10044 : else
10045 : {
10046 0 : if (SCALAR_FLOAT_TYPE_P (stept))
10047 : {
10048 0 : tree tem = build_int_cst (integer_type_node, nunits);
10049 0 : lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem);
10050 : }
10051 : else
10052 0 : lupdate_mul = build_int_cst (stept, nunits);
10053 0 : lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype,
10054 : lupdate_mul);
10055 : }
10056 11018 : for (; ivn < nvects; ++ivn)
10057 : {
10058 7612 : gimple *iv
10059 7612 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node)[ivn - nivs]);
10060 7612 : tree def = gimple_get_lhs (iv);
10061 7612 : if (ivn < 2*nivs)
10062 3504 : vec_steps[ivn - nivs]
10063 3504 : = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
10064 3504 : vec_steps[ivn - nivs], lupdate_mul);
10065 7612 : gimple_seq stmts = NULL;
10066 7612 : def = gimple_convert (&stmts, step_vectype, def);
10067 22836 : def = gimple_build (&stmts, PLUS_EXPR, step_vectype,
10068 7612 : def, vec_steps[ivn % nivs]);
10069 7612 : def = gimple_convert (&stmts, vectype, def);
10070 7612 : if (gimple_code (iv) == GIMPLE_PHI)
10071 3504 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10072 : else
10073 : {
10074 4108 : gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
10075 4108 : gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING);
10076 : }
10077 7612 : slp_node->push_vec_def (def);
10078 : }
10079 : }
10080 :
10081 15462 : new_bb = gsi_insert_seq_on_edge_immediate (pe, init_stmts);
10082 15462 : gcc_assert (!new_bb);
10083 :
10084 15462 : return true;
10085 15462 : }
10086 :
10087 : /* Function vectorizable_live_operation_1.
10088 :
10089 : helper function for vectorizable_live_operation. */
10090 :
10091 : static tree
10092 2881 : vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
10093 : tree vectype, slp_tree slp_node,
10094 : tree bitsize, tree bitstart, tree vec_lhs,
10095 : tree lhs_type, gimple_stmt_iterator *exit_gsi)
10096 : {
10097 2881 : gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10098 :
10099 2881 : tree vec_lhs_phi = copy_ssa_name (vec_lhs);
10100 2881 : gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
10101 8645 : for (unsigned i = 0; i < gimple_phi_num_args (phi); i++)
10102 2883 : SET_PHI_ARG_DEF (phi, i, vec_lhs);
10103 :
10104 2881 : gimple_seq stmts = NULL;
10105 2881 : tree new_tree;
10106 :
10107 : /* If bitstart is 0 then we can use a BIT_FIELD_REF */
10108 2881 : if (integer_zerop (bitstart))
10109 : {
10110 240 : tree scalar_res = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
10111 : vec_lhs_phi, bitsize, bitstart);
10112 :
10113 : /* Convert the extracted vector element to the scalar type. */
10114 240 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10115 : }
10116 2641 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10117 : {
10118 : /* Emit:
10119 :
10120 : SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>
10121 :
10122 : where VEC_LHS is the vectorized live-out result, LEN is the length of
10123 : the vector, BIAS is the load-store bias. The bias should not be used
10124 : at all since we are not using load/store operations, but LEN will be
10125 : REALLEN + BIAS, so subtract it to get to the correct position. */
10126 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10127 0 : gimple_seq tem = NULL;
10128 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10129 0 : tree len = vect_get_loop_len (loop_vinfo, &gsi,
10130 : &LOOP_VINFO_LENS (loop_vinfo),
10131 : 1, vectype, 0, 1, false);
10132 0 : gimple_seq_add_seq (&stmts, tem);
10133 :
10134 : /* LAST_INDEX = LEN - 1. */
10135 0 : tree last_index = gimple_build (&stmts, MINUS_EXPR, TREE_TYPE (len),
10136 0 : len, build_one_cst (TREE_TYPE (len)));
10137 :
10138 : /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>. */
10139 0 : tree scalar_res
10140 0 : = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
10141 : vec_lhs_phi, last_index);
10142 :
10143 : /* Convert the extracted vector element to the scalar type. */
10144 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10145 : }
10146 2641 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
10147 : {
10148 : /* Emit:
10149 :
10150 : SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
10151 :
10152 : where VEC_LHS is the vectorized live-out result and MASK is
10153 : the loop mask for the final iteration. */
10154 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10155 0 : tree scalar_type = TREE_TYPE (vectype);
10156 0 : gimple_seq tem = NULL;
10157 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10158 0 : tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
10159 : &LOOP_VINFO_MASKS (loop_vinfo),
10160 : 1, vectype, 0);
10161 0 : tree scalar_res;
10162 0 : gimple_seq_add_seq (&stmts, tem);
10163 :
10164 0 : scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type,
10165 : mask, vec_lhs_phi);
10166 :
10167 : /* Convert the extracted vector element to the scalar type. */
10168 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10169 : }
10170 : else
10171 : {
10172 2641 : tree bftype = TREE_TYPE (vectype);
10173 2641 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10174 85 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10175 2641 : new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart);
10176 2641 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10177 : &stmts, true, NULL_TREE);
10178 : }
10179 :
10180 2881 : *exit_gsi = gsi_after_labels (exit_bb);
10181 2881 : if (stmts)
10182 2881 : gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT);
10183 :
10184 2881 : return new_tree;
10185 : }
10186 :
10187 : /* Function vectorizable_live_operation.
10188 :
10189 : STMT_INFO computes a value that is used outside the loop. Check if
10190 : it can be supported. */
10191 :
10192 : bool
10193 308242 : vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
10194 : slp_tree slp_node, slp_instance slp_node_instance,
10195 : int slp_index, bool vec_stmt_p,
10196 : stmt_vector_for_cost *cost_vec)
10197 : {
10198 308242 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
10199 308242 : imm_use_iterator imm_iter;
10200 308242 : tree lhs, lhs_type, bitsize;
10201 308242 : tree vectype = SLP_TREE_VECTYPE (slp_node);
10202 308242 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
10203 308242 : gimple *use_stmt;
10204 308242 : use_operand_p use_p;
10205 308242 : auto_vec<tree> vec_oprnds;
10206 308242 : int vec_entry = 0;
10207 308242 : poly_uint64 vec_index = 0;
10208 :
10209 308242 : gcc_assert (STMT_VINFO_LIVE_P (stmt_info)
10210 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10211 :
10212 : /* If a stmt of a reduction is live, vectorize it via
10213 : vect_create_epilog_for_reduction. vectorizable_reduction assessed
10214 : validity so just trigger the transform here. */
10215 308242 : if (vect_is_reduction (slp_node))
10216 : {
10217 87334 : if (!vec_stmt_p)
10218 : {
10219 63819 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10220 63819 : return true;
10221 : }
10222 : /* For SLP reductions we vectorize the epilogue for all involved stmts
10223 : together. For SLP reduction chains we only get here once. */
10224 23515 : if (SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_group
10225 23240 : && slp_index != 0)
10226 : return true;
10227 23066 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
10228 23066 : if (VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
10229 23066 : || VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
10230 : return true;
10231 :
10232 22153 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10233 22153 : || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10234 22144 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
10235 : slp_node_instance,
10236 : LOOP_VINFO_MAIN_EXIT (loop_vinfo));
10237 :
10238 : /* If early break we only have to materialize the reduction on the merge
10239 : block, but we have to find an alternate exit first. */
10240 22153 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
10241 : {
10242 28 : slp_tree phis_node = slp_node_instance->reduc_phis;
10243 28 : stmt_info = SLP_TREE_REPRESENTATIVE (phis_node);
10244 89 : for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
10245 28 : if (exit != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
10246 : {
10247 23 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10248 : phis_node, slp_node_instance,
10249 : exit);
10250 23 : break;
10251 28 : }
10252 28 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10253 9 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10254 : phis_node, slp_node_instance,
10255 : LOOP_VINFO_MAIN_EXIT
10256 : (loop_vinfo));
10257 : }
10258 :
10259 : return true;
10260 : }
10261 :
10262 : /* If STMT is not relevant and it is a simple assignment and its inputs are
10263 : invariant then it can remain in place, unvectorized. The original last
10264 : scalar value that it computes will be used. */
10265 220908 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
10266 : {
10267 0 : gcc_assert (is_simple_and_all_uses_invariant (stmt_info, loop_vinfo));
10268 0 : if (dump_enabled_p ())
10269 0 : dump_printf_loc (MSG_NOTE, vect_location,
10270 : "statement is simple and uses invariant. Leaving in "
10271 : "place.\n");
10272 : return true;
10273 : }
10274 :
10275 220908 : gcc_assert (slp_index >= 0);
10276 :
10277 : /* Get the last occurrence of the scalar index from the concatenation of
10278 : all the slp vectors. Calculate which slp vector it is and the index
10279 : within. */
10280 220908 : int num_scalar = SLP_TREE_LANES (slp_node);
10281 220908 : int num_vec = vect_get_num_copies (vinfo, slp_node);
10282 220908 : poly_uint64 pos = (num_vec * nunits) - num_scalar + slp_index;
10283 :
10284 : /* Calculate which vector contains the result, and which lane of
10285 : that vector we need. */
10286 220908 : if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
10287 : {
10288 : if (dump_enabled_p ())
10289 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10290 : "Cannot determine which vector holds the"
10291 : " final result.\n");
10292 : return false;
10293 : }
10294 :
10295 220908 : if (!vec_stmt_p)
10296 : {
10297 : /* No transformation required. */
10298 175268 : if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
10299 : {
10300 28775 : if (SLP_TREE_LANES (slp_node) != 1)
10301 : {
10302 19 : if (dump_enabled_p ())
10303 19 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10304 : "can't operate on partial vectors "
10305 : "because an SLP statement is live after "
10306 : "the loop.\n");
10307 19 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10308 : }
10309 28756 : else if (num_vec > 1)
10310 : {
10311 16781 : if (dump_enabled_p ())
10312 53 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10313 : "can't operate on partial vectors "
10314 : "because ncopies is greater than 1.\n");
10315 16781 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10316 : }
10317 : else
10318 : {
10319 11975 : if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
10320 : OPTIMIZE_FOR_SPEED))
10321 0 : vect_record_loop_mask (loop_vinfo,
10322 : &LOOP_VINFO_MASKS (loop_vinfo),
10323 : 1, vectype, NULL);
10324 11975 : else if (can_vec_extract_var_idx_p (
10325 11975 : TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
10326 0 : vect_record_loop_len (loop_vinfo,
10327 : &LOOP_VINFO_LENS (loop_vinfo),
10328 : 1, vectype, 1);
10329 : else
10330 : {
10331 11975 : if (dump_enabled_p ())
10332 680 : dump_printf_loc (
10333 680 : MSG_MISSED_OPTIMIZATION, vect_location,
10334 : "can't operate on partial vectors "
10335 : "because the target doesn't support extract "
10336 : "last reduction.\n");
10337 11975 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10338 : }
10339 : }
10340 : }
10341 : /* ??? Enable for loop costing as well. */
10342 28775 : if (!loop_vinfo)
10343 101521 : record_stmt_cost (cost_vec, 1, vec_to_scalar, slp_node,
10344 : 0, vect_epilogue);
10345 175268 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10346 175268 : return true;
10347 : }
10348 :
10349 : /* Use the lhs of the original scalar statement. */
10350 45640 : gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
10351 45640 : if (dump_enabled_p ())
10352 1012 : dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
10353 : "stmt %G", stmt);
10354 :
10355 45640 : lhs = gimple_get_lhs (stmt);
10356 45640 : lhs_type = TREE_TYPE (lhs);
10357 :
10358 45640 : bitsize = vector_element_bits_tree (vectype);
10359 :
10360 : /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
10361 45640 : gcc_assert (!loop_vinfo
10362 : || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10363 : && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10364 : || SLP_TREE_LANES (slp_node) == 1));
10365 :
10366 : /* Get the correct slp vectorized stmt. */
10367 45640 : tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
10368 :
10369 : /* In case we need to early break vectorize also get the first stmt. */
10370 45640 : tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
10371 :
10372 : /* Get entry to use. */
10373 45640 : tree bitstart = bitsize_int (vec_index);
10374 45640 : bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
10375 :
10376 45640 : if (loop_vinfo)
10377 : {
10378 : /* Ensure the VEC_LHS for lane extraction stmts satisfy loop-closed PHI
10379 : requirement, insert one phi node for it. It looks like:
10380 : loop;
10381 : BB:
10382 : # lhs' = PHI <lhs>
10383 : ==>
10384 : loop;
10385 : BB:
10386 : # vec_lhs' = PHI <vec_lhs>
10387 : new_tree = lane_extract <vec_lhs', ...>;
10388 : lhs' = new_tree; */
10389 :
10390 2922 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10391 : /* Check if we have a loop where the chosen exit is not the main exit,
10392 : in these cases for an early break we restart the iteration the vector code
10393 : did. For the live values we want the value at the start of the iteration
10394 : rather than at the end. */
10395 2922 : edge main_e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
10396 2922 : bool all_exits_as_early_p = LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo);
10397 12251 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10398 9329 : if (!is_gimple_debug (use_stmt)
10399 9329 : && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
10400 2881 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10401 : {
10402 2881 : edge e = gimple_phi_arg_edge (as_a <gphi *> (use_stmt),
10403 2881 : phi_arg_index_from_use (use_p));
10404 2881 : gcc_assert (loop_exit_edge_p (loop, e));
10405 2881 : bool main_exit_edge = e == main_e;
10406 2881 : tree tmp_vec_lhs = vec_lhs;
10407 2881 : tree tmp_bitstart = bitstart;
10408 :
10409 : /* For early exit where the exit is not in the BB that leads
10410 : to the latch then we're restarting the iteration in the
10411 : scalar loop. So get the first live value. */
10412 2881 : bool early_break_first_element_p
10413 2881 : = all_exits_as_early_p || !main_exit_edge;
10414 2881 : if (early_break_first_element_p)
10415 : {
10416 222 : tmp_vec_lhs = vec_lhs0;
10417 222 : tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
10418 : }
10419 :
10420 2881 : gimple_stmt_iterator exit_gsi;
10421 2881 : tree new_tree
10422 2881 : = vectorizable_live_operation_1 (loop_vinfo,
10423 : e->dest, vectype,
10424 : slp_node, bitsize,
10425 : tmp_bitstart, tmp_vec_lhs,
10426 : lhs_type, &exit_gsi);
10427 :
10428 2881 : auto gsi = gsi_for_stmt (use_stmt);
10429 2881 : tree lhs_phi = gimple_phi_result (use_stmt);
10430 2881 : remove_phi_node (&gsi, false);
10431 2881 : gimple *copy = gimple_build_assign (lhs_phi, new_tree);
10432 2881 : gsi_insert_before (&exit_gsi, copy, GSI_SAME_STMT);
10433 2881 : break;
10434 2922 : }
10435 :
10436 : /* There a no further out-of-loop uses of lhs by LC-SSA construction. */
10437 9370 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10438 6448 : gcc_assert (is_gimple_debug (use_stmt)
10439 2922 : || flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)));
10440 : }
10441 : else
10442 : {
10443 : /* For basic-block vectorization simply insert the lane-extraction. */
10444 42718 : tree bftype = TREE_TYPE (vectype);
10445 42718 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10446 52 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10447 42718 : tree new_tree = build3 (BIT_FIELD_REF, bftype,
10448 : vec_lhs, bitsize, bitstart);
10449 42718 : gimple_seq stmts = NULL;
10450 42718 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10451 : &stmts, true, NULL_TREE);
10452 42718 : if (TREE_CODE (new_tree) == SSA_NAME
10453 85436 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
10454 2 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = 1;
10455 42718 : gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
10456 42718 : if (TREE_CODE (vec_lhs) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (vec_lhs))
10457 0 : vinfo->insert_seq_on_entry (stmt_info, stmts);
10458 42718 : else if (is_a <gphi *> (vec_stmt))
10459 : {
10460 2948 : gimple_stmt_iterator si = gsi_after_labels (gimple_bb (vec_stmt));
10461 2948 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10462 : }
10463 : else
10464 : {
10465 39770 : gimple_stmt_iterator si = gsi_for_stmt (vec_stmt);
10466 39770 : gsi_insert_seq_after (&si, stmts, GSI_SAME_STMT);
10467 : }
10468 :
10469 : /* Replace use of lhs with newly computed result. If the use stmt is a
10470 : single arg PHI, just replace all uses of PHI result. It's necessary
10471 : because lcssa PHI defining lhs may be before newly inserted stmt. */
10472 42718 : use_operand_p use_p;
10473 42718 : stmt_vec_info use_stmt_info;
10474 210614 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10475 167896 : if (!is_gimple_debug (use_stmt)
10476 167896 : && (!(use_stmt_info = vinfo->lookup_stmt (use_stmt))
10477 118757 : || !PURE_SLP_STMT (use_stmt_info)))
10478 : {
10479 : /* ??? This can happen when the live lane ends up being
10480 : rooted in a vector construction code-generated by an
10481 : external SLP node (and code-generation for that already
10482 : happened).
10483 : Doing this is what would happen if that vector CTOR
10484 : were not code-generated yet so it is not too bad.
10485 : ??? In fact we'd likely want to avoid this situation
10486 : in the first place. */
10487 72968 : if (TREE_CODE (new_tree) == SSA_NAME
10488 72968 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10489 72968 : && gimple_code (use_stmt) != GIMPLE_PHI
10490 134535 : && !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (new_tree),
10491 : use_stmt))
10492 : {
10493 0 : if (dump_enabled_p ())
10494 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10495 : "Using original scalar computation for "
10496 : "live lane because use precedes vector "
10497 : "def\n");
10498 0 : continue;
10499 : }
10500 153690 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10501 : {
10502 : /* ??? It can also happen that we end up pulling a def into
10503 : a loop where replacing out-of-loop uses would require
10504 : a new LC SSA PHI node. Retain the original scalar in
10505 : those cases as well. PR98064. */
10506 76845 : edge e;
10507 76845 : if (TREE_CODE (new_tree) == SSA_NAME
10508 76845 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10509 76845 : && TREE_CODE (vec_lhs) == SSA_NAME
10510 76845 : && !SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
10511 76845 : && (gimple_bb (use_stmt)->loop_father
10512 76845 : != gimple_bb (vec_stmt)->loop_father)
10513 : /* But a replacement in a LC PHI is OK. This happens
10514 : in gcc.dg/vect/bb-slp-57.c for example. */
10515 8556 : && (gimple_code (use_stmt) != GIMPLE_PHI
10516 3957 : || (((e = phi_arg_edge_from_use (use_p)), true)
10517 3957 : && !loop_exit_edge_p
10518 3957 : (gimple_bb (vec_stmt)->loop_father, e)))
10519 83122 : && !flow_loop_nested_p (gimple_bb (vec_stmt)->loop_father,
10520 6277 : gimple_bb (use_stmt)->loop_father))
10521 : {
10522 0 : if (dump_enabled_p ())
10523 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10524 : "Using original scalar computation for "
10525 : "live lane because there is an "
10526 : "out-of-loop definition for it\n");
10527 0 : continue;
10528 : }
10529 76845 : SET_USE (use_p, new_tree);
10530 : }
10531 72968 : update_stmt (use_stmt);
10532 42718 : }
10533 : }
10534 :
10535 : return true;
10536 308242 : }
10537 :
10538 : /* Given loop represented by LOOP_VINFO, return true if computation of
10539 : LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
10540 : otherwise. */
10541 :
10542 : static bool
10543 62101 : loop_niters_no_overflow (loop_vec_info loop_vinfo)
10544 : {
10545 62101 : gcc_assert (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo));
10546 :
10547 : /* Constant case. */
10548 62101 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
10549 : {
10550 36240 : tree cst_niters = LOOP_VINFO_NITERS (loop_vinfo);
10551 36240 : tree cst_nitersm1 = LOOP_VINFO_NITERSM1 (loop_vinfo);
10552 :
10553 36240 : gcc_assert (TREE_CODE (cst_niters) == INTEGER_CST);
10554 36240 : gcc_assert (TREE_CODE (cst_nitersm1) == INTEGER_CST);
10555 36240 : if (wi::to_widest (cst_nitersm1) < wi::to_widest (cst_niters))
10556 : return true;
10557 : }
10558 :
10559 25861 : widest_int max;
10560 25861 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10561 : /* Check the upper bound of loop niters. */
10562 25861 : if (get_max_loop_iterations (loop, &max))
10563 : {
10564 25861 : tree type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
10565 25861 : signop sgn = TYPE_SIGN (type);
10566 25861 : widest_int type_max = widest_int::from (wi::max_value (type), sgn);
10567 25861 : if (max < type_max)
10568 25686 : return true;
10569 25861 : }
10570 : return false;
10571 62101 : }
10572 :
10573 : /* Return a mask type with half the number of elements as OLD_TYPE,
10574 : given that it should have mode NEW_MODE. */
10575 :
10576 : tree
10577 4571 : vect_halve_mask_nunits (tree old_type, machine_mode new_mode)
10578 : {
10579 4571 : poly_uint64 nunits = exact_div (TYPE_VECTOR_SUBPARTS (old_type), 2);
10580 4571 : return build_truth_vector_type_for_mode (nunits, new_mode);
10581 : }
10582 :
10583 : /* Return a mask type with twice as many elements as OLD_TYPE,
10584 : given that it should have mode NEW_MODE. */
10585 :
10586 : tree
10587 6997 : vect_double_mask_nunits (tree old_type, machine_mode new_mode)
10588 : {
10589 6997 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (old_type) * 2;
10590 6997 : return build_truth_vector_type_for_mode (nunits, new_mode);
10591 : }
10592 :
10593 : /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
10594 : contain a sequence of NVECTORS masks that each control a vector of type
10595 : VECTYPE. If SCALAR_MASK is nonnull, the fully-masked loop would AND
10596 : these vector masks with the vector version of SCALAR_MASK. */
10597 :
10598 : void
10599 106150 : vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
10600 : unsigned int nvectors, tree vectype, tree scalar_mask)
10601 : {
10602 106150 : gcc_assert (nvectors != 0);
10603 :
10604 106150 : if (scalar_mask)
10605 : {
10606 4968 : scalar_cond_masked_key cond (scalar_mask, nvectors);
10607 4968 : loop_vinfo->scalar_cond_masked_set.add (cond);
10608 : }
10609 :
10610 106150 : masks->mask_set.add (std::make_pair (vectype, nvectors));
10611 106150 : }
10612 :
10613 : /* Given a complete set of masks MASKS, extract mask number INDEX
10614 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10615 : where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
10616 :
10617 : See the comment above vec_loop_masks for more details about the mask
10618 : arrangement. */
10619 :
10620 : tree
10621 214 : vect_get_loop_mask (loop_vec_info loop_vinfo,
10622 : gimple_stmt_iterator *gsi, vec_loop_masks *masks,
10623 : unsigned int nvectors, tree vectype, unsigned int index)
10624 : {
10625 214 : if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10626 : == vect_partial_vectors_while_ult)
10627 : {
10628 0 : rgroup_controls *rgm = &(masks->rgc_vec)[nvectors - 1];
10629 0 : tree mask_type = rgm->type;
10630 :
10631 : /* Populate the rgroup's mask array, if this is the first time we've
10632 : used it. */
10633 0 : if (rgm->controls.is_empty ())
10634 : {
10635 0 : rgm->controls.safe_grow_cleared (nvectors, true);
10636 0 : for (unsigned int i = 0; i < nvectors; ++i)
10637 : {
10638 0 : tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
10639 : /* Provide a dummy definition until the real one is available. */
10640 0 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10641 0 : rgm->controls[i] = mask;
10642 : }
10643 : }
10644 :
10645 0 : tree mask = rgm->controls[index];
10646 0 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
10647 0 : TYPE_VECTOR_SUBPARTS (vectype)))
10648 : {
10649 : /* A loop mask for data type X can be reused for data type Y
10650 : if X has N times more elements than Y and if Y's elements
10651 : are N times bigger than X's. In this case each sequence
10652 : of N elements in the loop mask will be all-zero or all-one.
10653 : We can then view-convert the mask so that each sequence of
10654 : N elements is replaced by a single element. */
10655 0 : gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type),
10656 : TYPE_VECTOR_SUBPARTS (vectype)));
10657 0 : gimple_seq seq = NULL;
10658 0 : mask_type = truth_type_for (vectype);
10659 0 : mask = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, mask);
10660 0 : if (seq)
10661 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10662 : }
10663 : return mask;
10664 : }
10665 214 : else if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10666 : == vect_partial_vectors_avx512)
10667 : {
10668 : /* The number of scalars per iteration and the number of vectors are
10669 : both compile-time constants. */
10670 214 : unsigned int nscalars_per_iter
10671 214 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10672 214 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10673 :
10674 214 : rgroup_controls *rgm = &masks->rgc_vec[nscalars_per_iter - 1];
10675 :
10676 : /* The stored nV is dependent on the mask type produced. */
10677 214 : gcc_assert (exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10678 : TYPE_VECTOR_SUBPARTS (rgm->type)).to_constant ()
10679 : == rgm->factor);
10680 214 : nvectors = rgm->factor;
10681 :
10682 : /* Populate the rgroup's mask array, if this is the first time we've
10683 : used it. */
10684 214 : if (rgm->controls.is_empty ())
10685 : {
10686 23 : rgm->controls.safe_grow_cleared (nvectors, true);
10687 135 : for (unsigned int i = 0; i < nvectors; ++i)
10688 : {
10689 89 : tree mask = make_temp_ssa_name (rgm->type, NULL, "loop_mask");
10690 : /* Provide a dummy definition until the real one is available. */
10691 89 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10692 89 : rgm->controls[i] = mask;
10693 : }
10694 : }
10695 214 : if (known_eq (TYPE_VECTOR_SUBPARTS (rgm->type),
10696 : TYPE_VECTOR_SUBPARTS (vectype)))
10697 166 : return rgm->controls[index];
10698 :
10699 : /* Split the vector if needed. Since we are dealing with integer mode
10700 : masks with AVX512 we can operate on the integer representation
10701 : performing the whole vector shifting. */
10702 48 : unsigned HOST_WIDE_INT factor;
10703 48 : bool ok = constant_multiple_p (TYPE_VECTOR_SUBPARTS (rgm->type),
10704 48 : TYPE_VECTOR_SUBPARTS (vectype), &factor);
10705 0 : gcc_assert (ok);
10706 48 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (rgm->type)) == MODE_INT);
10707 48 : tree mask_type = truth_type_for (vectype);
10708 48 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (mask_type)) == MODE_INT);
10709 48 : unsigned vi = index / factor;
10710 48 : unsigned vpart = index % factor;
10711 48 : tree vec = rgm->controls[vi];
10712 48 : gimple_seq seq = NULL;
10713 48 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR,
10714 48 : lang_hooks.types.type_for_mode
10715 48 : (TYPE_MODE (rgm->type), 1), vec);
10716 : /* For integer mode masks simply shift the right bits into position. */
10717 48 : if (vpart != 0)
10718 40 : vec = gimple_build (&seq, RSHIFT_EXPR, TREE_TYPE (vec), vec,
10719 : build_int_cst (integer_type_node,
10720 80 : (TYPE_VECTOR_SUBPARTS (vectype)
10721 40 : * vpart)));
10722 48 : vec = gimple_convert (&seq, lang_hooks.types.type_for_mode
10723 48 : (TYPE_MODE (mask_type), 1), vec);
10724 48 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, vec);
10725 48 : if (seq)
10726 48 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10727 : return vec;
10728 : }
10729 : else
10730 0 : gcc_unreachable ();
10731 : }
10732 :
10733 : /* Record that LOOP_VINFO would need LENS to contain a sequence of NVECTORS
10734 : lengths for controlling an operation on VECTYPE. The operation splits
10735 : each element of VECTYPE into FACTOR separate subelements, measuring the
10736 : length as a number of these subelements. */
10737 :
10738 : void
10739 0 : vect_record_loop_len (loop_vec_info loop_vinfo, vec_loop_lens *lens,
10740 : unsigned int nvectors, tree vectype, unsigned int factor)
10741 : {
10742 0 : gcc_assert (nvectors != 0);
10743 0 : if (lens->length () < nvectors)
10744 0 : lens->safe_grow_cleared (nvectors, true);
10745 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10746 :
10747 : /* The number of scalars per iteration, scalar occupied bytes and
10748 : the number of vectors are both compile-time constants. */
10749 0 : unsigned int nscalars_per_iter
10750 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10751 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10752 :
10753 0 : if (rgl->max_nscalars_per_iter < nscalars_per_iter)
10754 : {
10755 : /* For now, we only support cases in which all loads and stores fall back
10756 : to VnQI or none do. */
10757 0 : gcc_assert (!rgl->max_nscalars_per_iter
10758 : || (rgl->factor == 1 && factor == 1)
10759 : || (rgl->max_nscalars_per_iter * rgl->factor
10760 : == nscalars_per_iter * factor));
10761 0 : rgl->max_nscalars_per_iter = nscalars_per_iter;
10762 0 : rgl->type = vectype;
10763 0 : rgl->factor = factor;
10764 : }
10765 0 : }
10766 :
10767 : /* Given a complete set of lengths LENS, extract length number INDEX
10768 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10769 : where 0 <= INDEX < NVECTORS. Return a value that contains FACTOR
10770 : multiplied by the number of elements that should be processed.
10771 : Insert any set-up statements before GSI. */
10772 :
10773 : tree
10774 0 : vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10775 : vec_loop_lens *lens, unsigned int nvectors, tree vectype,
10776 : unsigned int index, unsigned int factor, bool adjusted)
10777 : {
10778 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10779 0 : bool use_bias_adjusted_len =
10780 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) != 0;
10781 :
10782 : /* Populate the rgroup's len array, if this is the first time we've
10783 : used it. */
10784 0 : if (rgl->controls.is_empty ())
10785 : {
10786 0 : rgl->controls.safe_grow_cleared (nvectors, true);
10787 0 : for (unsigned int i = 0; i < nvectors; ++i)
10788 : {
10789 0 : tree len_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
10790 0 : gcc_assert (len_type != NULL_TREE);
10791 :
10792 0 : tree len = make_temp_ssa_name (len_type, NULL, "loop_len");
10793 :
10794 : /* Provide a dummy definition until the real one is available. */
10795 0 : SSA_NAME_DEF_STMT (len) = gimple_build_nop ();
10796 0 : rgl->controls[i] = len;
10797 :
10798 0 : if (use_bias_adjusted_len)
10799 : {
10800 0 : gcc_assert (i == 0);
10801 0 : tree adjusted_len =
10802 0 : make_temp_ssa_name (len_type, NULL, "adjusted_loop_len");
10803 0 : SSA_NAME_DEF_STMT (adjusted_len) = gimple_build_nop ();
10804 0 : rgl->bias_adjusted_ctrl = adjusted_len;
10805 : }
10806 : }
10807 : }
10808 :
10809 0 : if (use_bias_adjusted_len && adjusted)
10810 0 : return rgl->bias_adjusted_ctrl;
10811 :
10812 0 : tree loop_len = rgl->controls[index];
10813 0 : if (rgl->factor == 1 && factor == 1)
10814 : {
10815 0 : poly_int64 nunits1 = TYPE_VECTOR_SUBPARTS (rgl->type);
10816 0 : poly_int64 nunits2 = TYPE_VECTOR_SUBPARTS (vectype);
10817 0 : if (maybe_ne (nunits1, nunits2))
10818 : {
10819 : /* A loop len for data type X can be reused for data type Y
10820 : if X has N times more elements than Y and if Y's elements
10821 : are N times bigger than X's. */
10822 0 : gcc_assert (multiple_p (nunits1, nunits2));
10823 0 : factor = exact_div (nunits1, nunits2).to_constant ();
10824 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10825 0 : gimple_seq seq = NULL;
10826 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10827 0 : build_int_cst (iv_type, factor));
10828 0 : if (seq)
10829 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10830 : }
10831 0 : }
10832 0 : else if (factor && rgl->factor != factor)
10833 : {
10834 : /* The number of scalars per iteration, scalar occupied bytes and
10835 : the number of vectors are both compile-time constants. */
10836 0 : unsigned int nscalars_per_iter
10837 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10838 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10839 0 : unsigned int rglvecsize = rgl->factor * rgl->max_nscalars_per_iter;
10840 0 : unsigned int vecsize = nscalars_per_iter * factor;
10841 0 : if (rglvecsize > vecsize)
10842 : {
10843 0 : unsigned int fac = rglvecsize / vecsize;
10844 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10845 0 : gimple_seq seq = NULL;
10846 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10847 0 : build_int_cst (iv_type, fac));
10848 0 : if (seq)
10849 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10850 : }
10851 0 : else if (rglvecsize < vecsize)
10852 : {
10853 0 : unsigned int fac = vecsize / rglvecsize;
10854 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10855 0 : gimple_seq seq = NULL;
10856 0 : loop_len = gimple_build (&seq, MULT_EXPR, iv_type, loop_len,
10857 0 : build_int_cst (iv_type, fac));
10858 0 : if (seq)
10859 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10860 : }
10861 : }
10862 : return loop_len;
10863 : }
10864 :
10865 : /* Generate the tree for the loop len mask and return it. Given the lens,
10866 : nvectors, vectype, index and factor to gen the len mask as below.
10867 :
10868 : tree len_mask = VCOND_MASK_LEN (compare_mask, ones, zero, len, bias)
10869 : */
10870 : tree
10871 0 : vect_gen_loop_len_mask (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10872 : gimple_stmt_iterator *cond_gsi, vec_loop_lens *lens,
10873 : unsigned int nvectors, tree vectype, tree stmt,
10874 : unsigned int index, unsigned int factor)
10875 : {
10876 0 : tree all_one_mask = build_all_ones_cst (vectype);
10877 0 : tree all_zero_mask = build_zero_cst (vectype);
10878 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens, nvectors, vectype, index,
10879 : factor, true);
10880 0 : tree bias = build_int_cst (intQI_type_node,
10881 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo));
10882 0 : tree len_mask = make_temp_ssa_name (TREE_TYPE (stmt), NULL, "vec_len_mask");
10883 0 : gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5, stmt,
10884 : all_one_mask, all_zero_mask, len,
10885 : bias);
10886 0 : gimple_call_set_lhs (call, len_mask);
10887 0 : gsi_insert_before (cond_gsi, call, GSI_SAME_STMT);
10888 :
10889 0 : return len_mask;
10890 : }
10891 :
10892 : /* Scale profiling counters by estimation for LOOP which is vectorized
10893 : by factor VF.
10894 : If FLAT is true, the loop we started with had unrealistically flat
10895 : profile. */
10896 :
10897 : static void
10898 62144 : scale_profile_for_vect_loop (class loop *loop, edge exit_e, unsigned vf, bool flat)
10899 : {
10900 : /* For flat profiles do not scale down proportionally by VF and only
10901 : cap by known iteration count bounds. */
10902 62144 : if (flat)
10903 : {
10904 34846 : if (dump_file && (dump_flags & TDF_DETAILS))
10905 5308 : fprintf (dump_file,
10906 : "Vectorized loop profile seems flat; not scaling iteration "
10907 : "count down by the vectorization factor %i\n", vf);
10908 34846 : scale_loop_profile (loop, profile_probability::always (),
10909 : get_likely_max_loop_iterations_int (loop));
10910 34846 : return;
10911 : }
10912 : /* Loop body executes VF fewer times and exit increases VF times. */
10913 27298 : profile_count entry_count = loop_preheader_edge (loop)->count ();
10914 :
10915 : /* If we have unreliable loop profile avoid dropping entry
10916 : count below header count. This can happen since loops
10917 : has unrealistically low trip counts. */
10918 27298 : while (vf > 1
10919 28375 : && loop->header->count > entry_count
10920 57782 : && loop->header->count < entry_count * vf)
10921 : {
10922 2109 : if (dump_file && (dump_flags & TDF_DETAILS))
10923 155 : fprintf (dump_file,
10924 : "Vectorization factor %i seems too large for profile "
10925 : "previously believed to be consistent; reducing.\n", vf);
10926 2109 : vf /= 2;
10927 : }
10928 :
10929 27298 : if (entry_count.nonzero_p ())
10930 27298 : set_edge_probability_and_rescale_others
10931 27298 : (exit_e,
10932 27298 : entry_count.probability_in (loop->header->count / vf));
10933 : /* Avoid producing very large exit probability when we do not have
10934 : sensible profile. */
10935 0 : else if (exit_e->probability < profile_probability::always () / (vf * 2))
10936 0 : set_edge_probability_and_rescale_others (exit_e, exit_e->probability * vf);
10937 27298 : loop->latch->count = single_pred_edge (loop->latch)->count ();
10938 :
10939 27298 : scale_loop_profile (loop, profile_probability::always () / vf,
10940 : get_likely_max_loop_iterations_int (loop));
10941 : }
10942 :
10943 : /* Update EPILOGUE's loop_vec_info. EPILOGUE was constructed as a copy of the
10944 : original loop that has now been vectorized.
10945 :
10946 : The inits of the data_references need to be advanced with the number of
10947 : iterations of the main loop. This has been computed in vect_do_peeling and
10948 : is stored in parameter ADVANCE.
10949 :
10950 : Since the loop_vec_info of this EPILOGUE was constructed for the original
10951 : loop, its stmt_vec_infos all point to the original statements. These need
10952 : to be updated to point to their corresponding copies.
10953 :
10954 : The data_reference's connections also need to be updated. Their
10955 : corresponding dr_vec_info need to be reconnected to the EPILOGUE's
10956 : stmt_vec_infos, their statements need to point to their corresponding
10957 : copy. */
10958 :
10959 : static void
10960 6854 : update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
10961 : {
10962 6854 : loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
10963 6854 : hash_map<tree,tree> mapping;
10964 6854 : gimple *orig_stmt, *new_stmt;
10965 6854 : gimple_stmt_iterator epilogue_gsi;
10966 6854 : gphi_iterator epilogue_phi_gsi;
10967 6854 : stmt_vec_info stmt_vinfo = NULL, related_vinfo;
10968 6854 : basic_block *epilogue_bbs = get_loop_body (epilogue);
10969 6854 : unsigned i;
10970 :
10971 6854 : free (LOOP_VINFO_BBS (epilogue_vinfo));
10972 6854 : LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
10973 6854 : LOOP_VINFO_NBBS (epilogue_vinfo) = epilogue->num_nodes;
10974 :
10975 : /* The EPILOGUE loop is a copy of the original loop so they share the same
10976 : gimple UIDs. In this loop we update the loop_vec_info of the EPILOGUE to
10977 : point to the copied statements. */
10978 20562 : for (unsigned i = 0; i < epilogue->num_nodes; ++i)
10979 : {
10980 13708 : for (epilogue_phi_gsi = gsi_start_phis (epilogue_bbs[i]);
10981 35338 : !gsi_end_p (epilogue_phi_gsi); gsi_next (&epilogue_phi_gsi))
10982 : {
10983 21630 : new_stmt = epilogue_phi_gsi.phi ();
10984 :
10985 21630 : gcc_assert (gimple_uid (new_stmt) > 0);
10986 21630 : stmt_vinfo
10987 21630 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
10988 :
10989 21630 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
10990 : }
10991 :
10992 27416 : for (epilogue_gsi = gsi_start_bb (epilogue_bbs[i]);
10993 137460 : !gsi_end_p (epilogue_gsi); gsi_next (&epilogue_gsi))
10994 : {
10995 123752 : new_stmt = gsi_stmt (epilogue_gsi);
10996 123752 : if (is_gimple_debug (new_stmt))
10997 20524 : continue;
10998 :
10999 103228 : gcc_assert (gimple_uid (new_stmt) > 0);
11000 103228 : stmt_vinfo
11001 103228 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
11002 :
11003 103228 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
11004 :
11005 103228 : related_vinfo = STMT_VINFO_RELATED_STMT (stmt_vinfo);
11006 103228 : if (related_vinfo != NULL && related_vinfo != stmt_vinfo)
11007 : {
11008 1932 : gimple *stmt = STMT_VINFO_STMT (related_vinfo);
11009 : /* Set BB such that the assert in
11010 : 'get_initial_defs_for_reduction' is able to determine that
11011 : the BB of the related stmt is inside this loop. */
11012 1932 : gimple_set_bb (stmt,
11013 : gimple_bb (new_stmt));
11014 1932 : related_vinfo = STMT_VINFO_RELATED_STMT (related_vinfo);
11015 1932 : gcc_assert (related_vinfo == NULL
11016 : || related_vinfo == stmt_vinfo);
11017 : }
11018 : }
11019 : }
11020 :
11021 6854 : struct data_reference *dr;
11022 6854 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (epilogue_vinfo);
11023 30986 : FOR_EACH_VEC_ELT (datarefs, i, dr)
11024 : {
11025 24132 : orig_stmt = DR_STMT (dr);
11026 24132 : gcc_assert (gimple_uid (orig_stmt) > 0);
11027 24132 : stmt_vinfo = epilogue_vinfo->stmt_vec_infos[gimple_uid (orig_stmt) - 1];
11028 24132 : DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo);
11029 : }
11030 :
11031 : /* Advance data_reference's with the number of iterations of the previous
11032 : loop and its prologue. */
11033 6854 : vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
11034 :
11035 : /* Remember the advancement made. */
11036 6854 : LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
11037 6854 : }
11038 :
11039 : /* When vectorizing early break statements instructions that happen before
11040 : the early break in the current BB need to be moved to after the early
11041 : break. This function deals with that and assumes that any validity
11042 : checks has already been performed.
11043 :
11044 : While moving the instructions if it encounters a VUSE or VDEF it then
11045 : corrects the VUSES as it moves the statements along. GDEST is the location
11046 : in which to insert the new statements. */
11047 :
11048 : static void
11049 1460 : move_early_exit_stmts (loop_vec_info loop_vinfo)
11050 : {
11051 1460 : DUMP_VECT_SCOPE ("move_early_exit_stmts");
11052 :
11053 1460 : if (LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).is_empty ())
11054 1200 : return;
11055 :
11056 : /* Move all stmts that need moving. */
11057 260 : basic_block dest_bb = LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo);
11058 260 : gimple_stmt_iterator dest_gsi = gsi_after_labels (dest_bb);
11059 :
11060 260 : tree last_seen_vuse = NULL_TREE;
11061 627 : for (gimple *stmt : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo))
11062 : {
11063 : /* We have to update crossed degenerate virtual PHIs. Simply
11064 : elide them. */
11065 367 : if (gphi *vphi = dyn_cast <gphi *> (stmt))
11066 : {
11067 7 : tree vdef = gimple_phi_result (vphi);
11068 7 : tree vuse = gimple_phi_arg_def (vphi, 0);
11069 7 : imm_use_iterator iter;
11070 7 : use_operand_p use_p;
11071 7 : gimple *use_stmt;
11072 23 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
11073 : {
11074 32 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11075 16 : SET_USE (use_p, vuse);
11076 7 : }
11077 7 : auto gsi = gsi_for_stmt (stmt);
11078 7 : remove_phi_node (&gsi, true);
11079 7 : last_seen_vuse = vuse;
11080 7 : continue;
11081 7 : }
11082 :
11083 : /* Check to see if statement is still required for vect or has been
11084 : elided. */
11085 360 : auto stmt_info = loop_vinfo->lookup_stmt (stmt);
11086 360 : if (!stmt_info)
11087 0 : continue;
11088 :
11089 360 : if (dump_enabled_p ())
11090 165 : dump_printf_loc (MSG_NOTE, vect_location, "moving stmt %G", stmt);
11091 :
11092 360 : gimple_stmt_iterator stmt_gsi = gsi_for_stmt (stmt);
11093 360 : gsi_move_before (&stmt_gsi, &dest_gsi, GSI_NEW_STMT);
11094 720 : last_seen_vuse = gimple_vuse (stmt);
11095 : }
11096 :
11097 : /* Update all the stmts with their new reaching VUSES. */
11098 815 : for (auto p : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
11099 : {
11100 245 : if (dump_enabled_p ())
11101 167 : dump_printf_loc (MSG_NOTE, vect_location,
11102 : "updating vuse to %T for load %G",
11103 : last_seen_vuse, p);
11104 245 : gimple_set_vuse (p, last_seen_vuse);
11105 245 : update_stmt (p);
11106 : }
11107 :
11108 : /* And update the LC PHIs on exits. */
11109 1313 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
11110 533 : if (!dominated_by_p (CDI_DOMINATORS, e->src, dest_bb))
11111 291 : if (gphi *phi = get_virtual_phi (e->dest))
11112 551 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, last_seen_vuse);
11113 : }
11114 :
11115 : /* Generate adjustment code for early break scalar IVs filling in the value
11116 : we created earlier on for LOOP_VINFO_EARLY_BRK_NITERS_VAR. */
11117 :
11118 : static void
11119 1460 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
11120 : {
11121 1460 : DUMP_VECT_SCOPE ("vect_update_ivs_after_vectorizer_for_early_breaks");
11122 :
11123 1460 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
11124 : /* If no peeling was done then we have no IV to update. */
11125 1460 : || !LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo))
11126 590 : return;
11127 :
11128 870 : tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
11129 870 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
11130 870 : tree ty_var = TREE_TYPE (phi_var);
11131 870 : auto loop = LOOP_VINFO_LOOP (loop_vinfo);
11132 870 : tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
11133 :
11134 : /* Remove the existing dummy GIMPLE statement and just keep the def. */
11135 870 : gimple *def = SSA_NAME_DEF_STMT (phi_var);
11136 870 : auto def_gsi = gsi_for_stmt (def);
11137 870 : gsi_remove (&def_gsi, true);
11138 :
11139 870 : auto induction_phi = create_phi_node (induc_var, loop->header);
11140 870 : tree induc_def = PHI_RESULT (induction_phi);
11141 :
11142 : /* Create the iv update inside the loop. */
11143 870 : gimple_seq init_stmts = NULL;
11144 870 : gimple_seq stmts = NULL;
11145 870 : gimple_seq iv_stmts = NULL;
11146 870 : tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
11147 :
11148 870 : tree iter_var;
11149 870 : if (POINTER_TYPE_P (ty_var))
11150 0 : iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
11151 : tree_iv_incr);
11152 : else
11153 : {
11154 870 : tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
11155 870 : iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
11156 : }
11157 :
11158 870 : tree init_var = build_zero_cst (ty_var);
11159 870 : if (niters_skip)
11160 0 : init_var = gimple_build (&init_stmts, MINUS_EXPR, ty_var, init_var,
11161 : gimple_convert (&init_stmts, ty_var, niters_skip));
11162 :
11163 870 : add_phi_arg (induction_phi, iter_var,
11164 : loop_latch_edge (loop), UNKNOWN_LOCATION);
11165 870 : add_phi_arg (induction_phi, init_var,
11166 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
11167 :
11168 : /* Find the first insertion point in the BB. */
11169 870 : auto pe = loop_preheader_edge (loop);
11170 :
11171 : /* If we've done any peeling, calculate the peeling adjustment needed to the
11172 : final IV. */
11173 870 : if (niters_skip)
11174 : {
11175 0 : tree induc_type = TREE_TYPE (induc_def);
11176 0 : tree s_induc_type = signed_type_for (induc_type);
11177 0 : induc_def = gimple_build (&iv_stmts, MAX_EXPR, s_induc_type,
11178 : gimple_convert (&iv_stmts, s_induc_type,
11179 : induc_def),
11180 : build_zero_cst (s_induc_type));
11181 0 : auto stmt = gimple_build_assign (phi_var,
11182 : gimple_convert (&iv_stmts, induc_type,
11183 : induc_def));
11184 0 : gimple_seq_add_stmt_without_update (&iv_stmts, stmt);
11185 0 : basic_block exit_bb = NULL;
11186 : /* Identify the early exit merge block. I wish we had stored this. */
11187 0 : for (auto e : get_loop_exit_edges (loop))
11188 0 : if (e != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
11189 : {
11190 0 : exit_bb = e->dest;
11191 0 : break;
11192 0 : }
11193 :
11194 0 : gcc_assert (exit_bb);
11195 0 : auto exit_gsi = gsi_after_labels (exit_bb);
11196 0 : gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
11197 : }
11198 : /* Write the init_stmts in the loop-preheader block. */
11199 870 : auto psi = gsi_last_nondebug_bb (pe->src);
11200 870 : gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
11201 :
11202 : /* Write the adjustments at the end of the iv increment. */
11203 870 : bool insert_after;
11204 870 : gimple_stmt_iterator incr_gsi;
11205 870 : vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo), &incr_gsi,
11206 : &insert_after);
11207 :
11208 870 : if (insert_after)
11209 0 : gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT);
11210 : else
11211 870 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT);
11212 : }
11213 :
11214 : /* Function vect_transform_loop.
11215 :
11216 : The analysis phase has determined that the loop is vectorizable.
11217 : Vectorize the loop - created vectorized stmts to replace the scalar
11218 : stmts in the loop, and update the loop exit condition.
11219 : Returns scalar epilogue loop if any. */
11220 :
11221 : class loop *
11222 62144 : vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
11223 : {
11224 62144 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11225 62144 : class loop *epilogue = NULL;
11226 62144 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
11227 62144 : int nbbs = loop->num_nodes;
11228 62144 : int i;
11229 62144 : tree niters_vector = NULL_TREE;
11230 62144 : tree step_vector = NULL_TREE;
11231 62144 : tree niters_vector_mult_vf = NULL_TREE;
11232 62144 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11233 62144 : unsigned int lowest_vf = constant_lower_bound (vf);
11234 62144 : gimple *stmt;
11235 62144 : bool check_profitability = false;
11236 62144 : unsigned int th;
11237 62144 : bool flat = maybe_flat_loop_profile (loop);
11238 62144 : bool uncounted_p = LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo);
11239 :
11240 62144 : DUMP_VECT_SCOPE ("vec_transform_loop");
11241 :
11242 62144 : if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11243 55290 : loop_vinfo->shared->check_datarefs ();
11244 :
11245 : /* Use the more conservative vectorization threshold. If the number
11246 : of iterations is constant assume the cost check has been performed
11247 : by our caller. If the threshold makes all loops profitable that
11248 : run at least the (estimated) vectorization factor number of times
11249 : checking is pointless, too. */
11250 62144 : th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
11251 62144 : if (vect_apply_runtime_profitability_check_p (loop_vinfo))
11252 : {
11253 18839 : if (dump_enabled_p ())
11254 178 : dump_printf_loc (MSG_NOTE, vect_location,
11255 : "Profitability threshold is %d loop iterations.\n",
11256 : th);
11257 : check_profitability = true;
11258 : }
11259 :
11260 : /* Make sure there exists a single-predecessor exit bb. Do this before
11261 : versioning. */
11262 62144 : edge e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
11263 62144 : if (! single_pred_p (e->dest) && !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11264 : {
11265 19058 : split_loop_exit_edge (e, true);
11266 19058 : if (dump_enabled_p ())
11267 2275 : dump_printf (MSG_NOTE, "split exit edge\n");
11268 : }
11269 :
11270 : /* Version the loop first, if required, so the profitability check
11271 : comes first. */
11272 :
11273 62144 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
11274 : {
11275 3779 : class loop *sloop
11276 3779 : = vect_loop_versioning (loop_vinfo, loop_vectorized_call);
11277 3779 : sloop->force_vectorize = false;
11278 3779 : check_profitability = false;
11279 : }
11280 :
11281 : /* Make sure there exists a single-predecessor exit bb also on the
11282 : scalar loop copy. Do this after versioning but before peeling
11283 : so CFG structure is fine for both scalar and if-converted loop
11284 : to make slpeel_duplicate_current_defs_from_edges face matched
11285 : loop closed PHI nodes on the exit. */
11286 62144 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
11287 : {
11288 8088 : e = LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo);
11289 8088 : if (! single_pred_p (e->dest))
11290 : {
11291 7825 : split_loop_exit_edge (e, true);
11292 7825 : if (dump_enabled_p ())
11293 1150 : dump_printf (MSG_NOTE, "split exit edge of scalar loop\n");
11294 : }
11295 : }
11296 :
11297 62144 : tree niters = vect_build_loop_niters (loop_vinfo);
11298 62144 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = niters;
11299 62144 : tree nitersm1 = unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo));
11300 62144 : tree advance;
11301 62144 : drs_init_vec orig_drs_init;
11302 62144 : bool niters_no_overflow = uncounted_p ? false /* Not known. */
11303 62101 : : loop_niters_no_overflow (loop_vinfo);
11304 :
11305 62144 : epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
11306 : &step_vector, &niters_vector_mult_vf, th,
11307 : check_profitability, niters_no_overflow,
11308 : &advance);
11309 :
11310 62144 : LOOP_VINFO_IV_INCREMENT (loop_vinfo)
11311 62144 : = vect_get_loop_iv_increment (loop_vinfo);
11312 :
11313 : /* Assign hierarchical discriminators to the vectorized loop. */
11314 62144 : poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11315 62144 : unsigned int vf_int = constant_lower_bound (vf_val);
11316 62144 : if (vf_int > DISCR_MULTIPLICITY_MAX)
11317 : vf_int = DISCR_MULTIPLICITY_MAX;
11318 :
11319 : /* Assign unique copy_id dynamically instead of using hardcoded constants.
11320 : Epilogue and main vectorized loops get different copy_ids. */
11321 62144 : gimple *loop_last = last_nondebug_stmt (loop->header);
11322 62144 : location_t loop_loc
11323 62144 : = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION;
11324 61866 : if (loop_loc != UNKNOWN_LOCATION)
11325 : {
11326 51254 : unsigned int copyid = allocate_copyid_base (loop_loc, 1);
11327 51254 : assign_discriminators_to_loop (loop, vf_int, copyid);
11328 : }
11329 62144 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
11330 62144 : && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
11331 : {
11332 : /* Ifcvt duplicates loop preheader, loop body and produces an basic
11333 : block after loop exit. We need to scale all that. */
11334 90 : basic_block preheader
11335 90 : = loop_preheader_edge (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))->src;
11336 90 : preheader->count
11337 : = preheader->count.apply_probability
11338 90 : (LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11339 90 : scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo),
11340 : LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11341 90 : LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo)->dest->count = preheader->count;
11342 : }
11343 :
11344 62144 : if (niters_vector == NULL_TREE && !uncounted_p)
11345 : {
11346 28310 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
11347 28310 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11348 57413 : && known_eq (lowest_vf, vf))
11349 : {
11350 28305 : niters_vector
11351 28305 : = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
11352 28305 : LOOP_VINFO_INT_NITERS (loop_vinfo) / lowest_vf);
11353 28305 : step_vector = build_one_cst (TREE_TYPE (niters));
11354 : }
11355 803 : else if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11356 2 : vect_gen_vector_loop_niters (loop_vinfo, niters, &niters_vector,
11357 : &step_vector, niters_no_overflow);
11358 : else
11359 : /* vect_do_peeling subtracted the number of peeled prologue
11360 : iterations from LOOP_VINFO_NITERS. */
11361 801 : vect_gen_vector_loop_niters (loop_vinfo, LOOP_VINFO_NITERS (loop_vinfo),
11362 : &niters_vector, &step_vector,
11363 : niters_no_overflow);
11364 : }
11365 :
11366 : /* 1) Make sure the loop header has exactly two entries
11367 : 2) Make sure we have a preheader basic block. */
11368 :
11369 62144 : gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
11370 :
11371 62144 : split_edge (loop_preheader_edge (loop));
11372 :
11373 62144 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11374 : /* This will deal with any possible peeling. */
11375 2 : vect_prepare_for_masked_peels (loop_vinfo);
11376 :
11377 : /* Handle any code motion that we need to for early-break vectorization after
11378 : we've done peeling but just before we start vectorizing. */
11379 62144 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11380 : {
11381 1460 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vinfo);
11382 1460 : move_early_exit_stmts (loop_vinfo);
11383 : }
11384 :
11385 : /* Remove existing clobber stmts and prefetches. */
11386 189797 : for (i = 0; i < nbbs; i++)
11387 : {
11388 127653 : basic_block bb = bbs[i];
11389 1102912 : for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);)
11390 : {
11391 847606 : stmt = gsi_stmt (si);
11392 847606 : if (gimple_clobber_p (stmt)
11393 847606 : || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH))
11394 : {
11395 91 : unlink_stmt_vdef (stmt);
11396 91 : gsi_remove (&si, true);
11397 91 : release_defs (stmt);
11398 : }
11399 : else
11400 847515 : gsi_next (&si);
11401 : }
11402 : }
11403 :
11404 : /* Schedule the SLP instances. */
11405 62144 : if (!loop_vinfo->slp_instances.is_empty ())
11406 : {
11407 62144 : DUMP_VECT_SCOPE ("scheduling SLP instances");
11408 62144 : vect_schedule_slp (loop_vinfo, LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
11409 : false);
11410 : }
11411 :
11412 : /* Generate the loop invariant statements. */
11413 62144 : if (!gimple_seq_empty_p (LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo)))
11414 : {
11415 73 : if (dump_enabled_p ())
11416 30 : dump_printf_loc (MSG_NOTE, vect_location,
11417 : "------>generating loop invariant statements\n");
11418 73 : gimple_stmt_iterator gsi;
11419 73 : gsi = gsi_after_labels (loop_preheader_edge (loop)->src);
11420 73 : gsi_insert_seq_before (&gsi, LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo),
11421 : GSI_CONTINUE_LINKING);
11422 : }
11423 :
11424 : /* Stub out scalar statements that must not survive vectorization and
11425 : were not picked as relevant in any SLP instance.
11426 : Doing this here helps with grouped statements, or statements that
11427 : are involved in patterns. */
11428 189797 : for (i = 0; i < nbbs; i++)
11429 : {
11430 127653 : basic_block bb = bbs[i];
11431 127653 : stmt_vec_info stmt_info;
11432 255306 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
11433 1687684 : !gsi_end_p (gsi); gsi_next (&gsi))
11434 : {
11435 1560031 : gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
11436 6254 : if (!call || !gimple_call_internal_p (call))
11437 1554933 : continue;
11438 5098 : internal_fn ifn = gimple_call_internal_fn (call);
11439 5098 : if (ifn == IFN_MASK_LOAD)
11440 : {
11441 682 : tree lhs = gimple_get_lhs (call);
11442 682 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11443 : {
11444 0 : tree zero = build_zero_cst (TREE_TYPE (lhs));
11445 0 : gimple *new_stmt = gimple_build_assign (lhs, zero);
11446 0 : gsi_replace (&gsi, new_stmt, true);
11447 : }
11448 : }
11449 4416 : else if (conditional_internal_fn_code (ifn) != ERROR_MARK)
11450 : {
11451 2297 : tree lhs = gimple_get_lhs (call);
11452 2297 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11453 : {
11454 0 : tree else_arg
11455 0 : = gimple_call_arg (call, gimple_call_num_args (call) - 1);
11456 0 : gimple *new_stmt = gimple_build_assign (lhs, else_arg);
11457 0 : gsi_replace (&gsi, new_stmt, true);
11458 : }
11459 : }
11460 2119 : else if (ifn == IFN_MASK_CALL
11461 4 : && (stmt_info = loop_vinfo->lookup_stmt (call))
11462 4 : && !STMT_VINFO_RELEVANT_P (stmt_info)
11463 2123 : && !STMT_VINFO_LIVE_P (stmt_info))
11464 : {
11465 4 : gcc_assert (!gimple_call_lhs (stmt_info->stmt));
11466 4 : loop_vinfo->remove_stmt (stmt_info);
11467 : }
11468 : }
11469 : }
11470 :
11471 62144 : if (!uncounted_p)
11472 : {
11473 : /* The vectorization factor is always > 1, so if we use an IV increment of
11474 : 1. A zero NITERS becomes a nonzero NITERS_VECTOR. */
11475 62101 : if (integer_onep (step_vector))
11476 62080 : niters_no_overflow = true;
11477 :
11478 62101 : vect_set_loop_condition (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
11479 : loop_vinfo, niters_vector, step_vector,
11480 : niters_vector_mult_vf, !niters_no_overflow);
11481 : }
11482 :
11483 62144 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
11484 :
11485 : /* True if the final iteration might not handle a full vector's
11486 : worth of scalar iterations. */
11487 124288 : bool final_iter_may_be_partial
11488 62144 : = LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11489 62144 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo);
11490 :
11491 : /* +1 to convert latch counts to loop iteration counts. */
11492 62144 : int bias_for_lowest = 1;
11493 :
11494 : /* When we are peeling for gaps then we take away one scalar iteration
11495 : from the vector loop. Thus we can adjust the upper bound by one
11496 : scalar iteration. But only when we know the bound applies to the
11497 : IV exit test which might not be true when we have multiple exits. */
11498 62144 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11499 120978 : bias_for_lowest -= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
11500 :
11501 62144 : int bias_for_assumed = bias_for_lowest;
11502 62144 : int alignment_npeels = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
11503 62144 : if (alignment_npeels && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
11504 : {
11505 : /* When the amount of peeling is known at compile time, the first
11506 : iteration will have exactly alignment_npeels active elements.
11507 : In the worst case it will have at least one. */
11508 2 : int min_first_active = (alignment_npeels > 0 ? alignment_npeels : 1);
11509 2 : bias_for_lowest += lowest_vf - min_first_active;
11510 2 : bias_for_assumed += assumed_vf - min_first_active;
11511 : }
11512 : /* In these calculations the "- 1" converts loop iteration counts
11513 : back to latch counts. */
11514 62144 : if (loop->any_upper_bound)
11515 : {
11516 62128 : loop_vec_info main_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
11517 62128 : loop->nb_iterations_upper_bound
11518 62128 : = (final_iter_may_be_partial
11519 63591 : ? wi::udiv_ceil (loop->nb_iterations_upper_bound + bias_for_lowest,
11520 2926 : lowest_vf) - 1
11521 60665 : : wi::udiv_floor (loop->nb_iterations_upper_bound + bias_for_lowest,
11522 121330 : lowest_vf) - 1);
11523 62128 : if (main_vinfo
11524 : /* Both peeling for alignment and peeling for gaps can end up
11525 : with the scalar epilogue running for more than VF-1 iterations. */
11526 6854 : && !main_vinfo->peeling_for_alignment
11527 6806 : && !main_vinfo->peeling_for_gaps)
11528 : {
11529 6624 : unsigned int bound;
11530 6624 : poly_uint64 main_iters
11531 6624 : = upper_bound (LOOP_VINFO_VECT_FACTOR (main_vinfo),
11532 : LOOP_VINFO_COST_MODEL_THRESHOLD (main_vinfo));
11533 6624 : main_iters
11534 6624 : = upper_bound (main_iters,
11535 6624 : LOOP_VINFO_VERSIONING_THRESHOLD (main_vinfo));
11536 6624 : if (can_div_away_from_zero_p (main_iters,
11537 6624 : LOOP_VINFO_VECT_FACTOR (loop_vinfo),
11538 : &bound))
11539 6624 : loop->nb_iterations_upper_bound
11540 6624 : = wi::umin ((bound_wide_int) (bound - 1),
11541 6624 : loop->nb_iterations_upper_bound);
11542 : }
11543 : }
11544 62144 : if (loop->any_likely_upper_bound)
11545 62128 : loop->nb_iterations_likely_upper_bound
11546 62128 : = (final_iter_may_be_partial
11547 63591 : ? wi::udiv_ceil (loop->nb_iterations_likely_upper_bound
11548 1463 : + bias_for_lowest, lowest_vf) - 1
11549 60665 : : wi::udiv_floor (loop->nb_iterations_likely_upper_bound
11550 62128 : + bias_for_lowest, lowest_vf) - 1);
11551 62144 : if (loop->any_estimate)
11552 35847 : loop->nb_iterations_estimate
11553 35847 : = (final_iter_may_be_partial
11554 36544 : ? wi::udiv_ceil (loop->nb_iterations_estimate + bias_for_assumed,
11555 1394 : assumed_vf) - 1
11556 35150 : : wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
11557 70997 : assumed_vf) - 1);
11558 62144 : scale_profile_for_vect_loop (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
11559 : assumed_vf, flat);
11560 :
11561 62144 : if (dump_enabled_p ())
11562 : {
11563 11041 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11564 : {
11565 9591 : dump_printf_loc (MSG_NOTE, vect_location,
11566 : "LOOP VECTORIZED\n");
11567 9591 : if (loop->inner)
11568 345 : dump_printf_loc (MSG_NOTE, vect_location,
11569 : "OUTER LOOP VECTORIZED\n");
11570 9591 : dump_printf (MSG_NOTE, "\n");
11571 : }
11572 : else
11573 1450 : dump_printf_loc (MSG_NOTE, vect_location,
11574 : "LOOP EPILOGUE VECTORIZED (MODE=%s)\n",
11575 1450 : GET_MODE_NAME (loop_vinfo->vector_mode));
11576 : }
11577 :
11578 : /* Loops vectorized with a variable factor won't benefit from
11579 : unrolling/peeling. */
11580 62144 : if (!vf.is_constant ())
11581 : {
11582 : loop->unroll = 1;
11583 : if (dump_enabled_p ())
11584 : dump_printf_loc (MSG_NOTE, vect_location, "Disabling unrolling due to"
11585 : " variable-length vectorization factor\n");
11586 : }
11587 :
11588 : /* When we have unrolled the loop due to a user requested value we should
11589 : leave it up to the RTL unroll heuristics to determine if it's still worth
11590 : while to unroll more. */
11591 62144 : if (LOOP_VINFO_USER_UNROLL (loop_vinfo))
11592 44 : loop->unroll = 0;
11593 :
11594 : /* Free SLP instances here because otherwise stmt reference counting
11595 : won't work. */
11596 62144 : slp_instance instance;
11597 152615 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
11598 90471 : vect_free_slp_instance (instance);
11599 62144 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
11600 : /* Clear-up safelen field since its value is invalid after vectorization
11601 : since vectorized loop can have loop-carried dependencies. */
11602 62144 : loop->safelen = 0;
11603 :
11604 62144 : if (epilogue)
11605 : {
11606 : /* Accumulate past advancements made. */
11607 6854 : if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
11608 75 : advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
11609 : LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
11610 : advance);
11611 6854 : update_epilogue_loop_vinfo (epilogue, advance);
11612 :
11613 6854 : epilogue->simduid = loop->simduid;
11614 6854 : epilogue->force_vectorize = loop->force_vectorize;
11615 6854 : epilogue->dont_vectorize = false;
11616 : }
11617 :
11618 62144 : return epilogue;
11619 62144 : }
11620 :
11621 : /* The code below is trying to perform simple optimization - revert
11622 : if-conversion for masked stores, i.e. if the mask of a store is zero
11623 : do not perform it and all stored value producers also if possible.
11624 : For example,
11625 : for (i=0; i<n; i++)
11626 : if (c[i])
11627 : {
11628 : p1[i] += 1;
11629 : p2[i] = p3[i] +2;
11630 : }
11631 : this transformation will produce the following semi-hammock:
11632 :
11633 : if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
11634 : {
11635 : vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
11636 : vect__12.22_172 = vect__11.19_170 + vect_cst__171;
11637 : MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
11638 : vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
11639 : vect__19.28_184 = vect__18.25_182 + vect_cst__183;
11640 : MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
11641 : }
11642 : */
11643 :
11644 : void
11645 486 : optimize_mask_stores (class loop *loop)
11646 : {
11647 486 : basic_block *bbs = get_loop_body (loop);
11648 486 : unsigned nbbs = loop->num_nodes;
11649 486 : unsigned i;
11650 486 : basic_block bb;
11651 486 : class loop *bb_loop;
11652 486 : gimple_stmt_iterator gsi;
11653 486 : gimple *stmt;
11654 486 : auto_vec<gimple *> worklist;
11655 486 : auto_purge_vect_location sentinel;
11656 :
11657 486 : vect_location = find_loop_location (loop);
11658 : /* Pick up all masked stores in loop if any. */
11659 1944 : for (i = 0; i < nbbs; i++)
11660 : {
11661 972 : bb = bbs[i];
11662 16527 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
11663 14583 : gsi_next (&gsi))
11664 : {
11665 14583 : stmt = gsi_stmt (gsi);
11666 14583 : if (gimple_call_internal_p (stmt, IFN_MASK_STORE))
11667 658 : worklist.safe_push (stmt);
11668 : }
11669 : }
11670 :
11671 486 : free (bbs);
11672 486 : if (worklist.is_empty ())
11673 68 : return;
11674 :
11675 : /* Loop has masked stores. */
11676 1059 : while (!worklist.is_empty ())
11677 : {
11678 641 : gimple *last, *last_store;
11679 641 : edge e, efalse;
11680 641 : tree mask;
11681 641 : basic_block store_bb, join_bb;
11682 641 : gimple_stmt_iterator gsi_to;
11683 641 : tree vdef, new_vdef;
11684 641 : gphi *phi;
11685 641 : tree vectype;
11686 641 : tree zero;
11687 :
11688 641 : last = worklist.pop ();
11689 641 : mask = gimple_call_arg (last, 2);
11690 641 : bb = gimple_bb (last);
11691 : /* Create then_bb and if-then structure in CFG, then_bb belongs to
11692 : the same loop as if_bb. It could be different to LOOP when two
11693 : level loop-nest is vectorized and mask_store belongs to the inner
11694 : one. */
11695 641 : e = split_block (bb, last);
11696 641 : bb_loop = bb->loop_father;
11697 641 : gcc_assert (loop == bb_loop || flow_loop_nested_p (loop, bb_loop));
11698 641 : join_bb = e->dest;
11699 641 : store_bb = create_empty_bb (bb);
11700 641 : add_bb_to_loop (store_bb, bb_loop);
11701 641 : e->flags = EDGE_TRUE_VALUE;
11702 641 : efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
11703 : /* Put STORE_BB to likely part. */
11704 641 : efalse->probability = profile_probability::likely ();
11705 641 : e->probability = efalse->probability.invert ();
11706 641 : store_bb->count = efalse->count ();
11707 641 : make_single_succ_edge (store_bb, join_bb, EDGE_FALLTHRU);
11708 641 : if (dom_info_available_p (CDI_DOMINATORS))
11709 641 : set_immediate_dominator (CDI_DOMINATORS, store_bb, bb);
11710 641 : if (dump_enabled_p ())
11711 326 : dump_printf_loc (MSG_NOTE, vect_location,
11712 : "Create new block %d to sink mask stores.",
11713 : store_bb->index);
11714 : /* Create vector comparison with boolean result. */
11715 641 : vectype = TREE_TYPE (mask);
11716 641 : zero = build_zero_cst (vectype);
11717 641 : stmt = gimple_build_cond (EQ_EXPR, mask, zero, NULL_TREE, NULL_TREE);
11718 641 : gsi = gsi_last_bb (bb);
11719 641 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
11720 : /* Create new PHI node for vdef of the last masked store:
11721 : .MEM_2 = VDEF <.MEM_1>
11722 : will be converted to
11723 : .MEM.3 = VDEF <.MEM_1>
11724 : and new PHI node will be created in join bb
11725 : .MEM_2 = PHI <.MEM_1, .MEM_3>
11726 : */
11727 641 : vdef = gimple_vdef (last);
11728 641 : new_vdef = make_ssa_name (gimple_vop (cfun), last);
11729 641 : gimple_set_vdef (last, new_vdef);
11730 641 : phi = create_phi_node (vdef, join_bb);
11731 641 : add_phi_arg (phi, new_vdef, EDGE_SUCC (store_bb, 0), UNKNOWN_LOCATION);
11732 :
11733 : /* Put all masked stores with the same mask to STORE_BB if possible. */
11734 675 : while (true)
11735 : {
11736 658 : gimple_stmt_iterator gsi_from;
11737 658 : gimple *stmt1 = NULL;
11738 :
11739 : /* Move masked store to STORE_BB. */
11740 658 : last_store = last;
11741 658 : gsi = gsi_for_stmt (last);
11742 658 : gsi_from = gsi;
11743 : /* Shift GSI to the previous stmt for further traversal. */
11744 658 : gsi_prev (&gsi);
11745 658 : gsi_to = gsi_start_bb (store_bb);
11746 658 : gsi_move_before (&gsi_from, &gsi_to);
11747 : /* Setup GSI_TO to the non-empty block start. */
11748 658 : gsi_to = gsi_start_bb (store_bb);
11749 658 : if (dump_enabled_p ())
11750 342 : dump_printf_loc (MSG_NOTE, vect_location,
11751 : "Move stmt to created bb\n%G", last);
11752 : /* Move all stored value producers if possible. */
11753 4929 : while (!gsi_end_p (gsi))
11754 : {
11755 4928 : tree lhs;
11756 4928 : imm_use_iterator imm_iter;
11757 4928 : use_operand_p use_p;
11758 4928 : bool res;
11759 :
11760 : /* Skip debug statements. */
11761 4928 : if (is_gimple_debug (gsi_stmt (gsi)))
11762 : {
11763 1 : gsi_prev (&gsi);
11764 3088 : continue;
11765 : }
11766 4927 : stmt1 = gsi_stmt (gsi);
11767 : /* Do not consider statements writing to memory or having
11768 : volatile operand. */
11769 9679 : if (gimple_vdef (stmt1)
11770 9679 : || gimple_has_volatile_ops (stmt1))
11771 : break;
11772 4752 : gsi_from = gsi;
11773 4752 : gsi_prev (&gsi);
11774 4752 : lhs = gimple_get_lhs (stmt1);
11775 4752 : if (!lhs)
11776 : break;
11777 :
11778 : /* LHS of vectorized stmt must be SSA_NAME. */
11779 4752 : if (TREE_CODE (lhs) != SSA_NAME)
11780 : break;
11781 :
11782 4752 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11783 : {
11784 : /* Remove dead scalar statement. */
11785 3403 : if (has_zero_uses (lhs))
11786 : {
11787 3087 : gsi_remove (&gsi_from, true);
11788 3087 : release_defs (stmt1);
11789 3087 : continue;
11790 : }
11791 : }
11792 :
11793 : /* Check that LHS does not have uses outside of STORE_BB. */
11794 1665 : res = true;
11795 2872 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
11796 : {
11797 1689 : gimple *use_stmt;
11798 1689 : use_stmt = USE_STMT (use_p);
11799 1689 : if (is_gimple_debug (use_stmt))
11800 0 : continue;
11801 1689 : if (gimple_bb (use_stmt) != store_bb)
11802 : {
11803 : res = false;
11804 : break;
11805 : }
11806 1665 : }
11807 1665 : if (!res)
11808 : break;
11809 :
11810 1183 : if (gimple_vuse (stmt1)
11811 1645 : && gimple_vuse (stmt1) != gimple_vuse (last_store))
11812 : break;
11813 :
11814 : /* Can move STMT1 to STORE_BB. */
11815 1183 : if (dump_enabled_p ())
11816 618 : dump_printf_loc (MSG_NOTE, vect_location,
11817 : "Move stmt to created bb\n%G", stmt1);
11818 1183 : gsi_move_before (&gsi_from, &gsi_to);
11819 : /* Shift GSI_TO for further insertion. */
11820 1183 : gsi_prev (&gsi_to);
11821 : }
11822 : /* Put other masked stores with the same mask to STORE_BB. */
11823 658 : if (worklist.is_empty ()
11824 240 : || gimple_call_arg (worklist.last (), 2) != mask
11825 17 : || worklist.last () != stmt1)
11826 : break;
11827 17 : last = worklist.pop ();
11828 17 : }
11829 1282 : add_phi_arg (phi, gimple_vuse (last_store), e, UNKNOWN_LOCATION);
11830 : }
11831 486 : }
11832 :
11833 : /* Decide whether it is possible to use a zero-based induction variable
11834 : when vectorizing LOOP_VINFO with partial vectors. If it is, return
11835 : the value that the induction variable must be able to hold in order
11836 : to ensure that the rgroups eventually have no active vector elements.
11837 : Return -1 otherwise. */
11838 :
11839 : widest_int
11840 46964 : vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo)
11841 : {
11842 46964 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
11843 46964 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11844 46964 : unsigned HOST_WIDE_INT max_vf = vect_max_vf (loop_vinfo);
11845 :
11846 : /* Calculate the value that the induction variable must be able
11847 : to hit in order to ensure that we end the loop with an all-false mask.
11848 : This involves adding the maximum number of inactive trailing scalar
11849 : iterations. */
11850 46964 : widest_int iv_limit = -1;
11851 46964 : if (max_loop_iterations (loop, &iv_limit))
11852 : {
11853 46964 : if (niters_skip)
11854 : {
11855 : /* Add the maximum number of skipped iterations to the
11856 : maximum iteration count. */
11857 0 : if (TREE_CODE (niters_skip) == INTEGER_CST)
11858 0 : iv_limit += wi::to_widest (niters_skip);
11859 : else
11860 0 : iv_limit += max_vf - 1;
11861 : }
11862 46964 : else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
11863 : /* Make a conservatively-correct assumption. */
11864 326 : iv_limit += max_vf - 1;
11865 :
11866 : /* IV_LIMIT is the maximum number of latch iterations, which is also
11867 : the maximum in-range IV value. Round this value down to the previous
11868 : vector alignment boundary and then add an extra full iteration. */
11869 46964 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11870 46964 : iv_limit = (iv_limit & -(int) known_alignment (vf)) + max_vf;
11871 : }
11872 46964 : return iv_limit;
11873 : }
11874 :
11875 : /* For the given rgroup_controls RGC, check whether an induction variable
11876 : would ever hit a value that produces a set of all-false masks or zero
11877 : lengths before wrapping around. Return true if it's possible to wrap
11878 : around before hitting the desirable value, otherwise return false. */
11879 :
11880 : bool
11881 0 : vect_rgroup_iv_might_wrap_p (loop_vec_info loop_vinfo, rgroup_controls *rgc)
11882 : {
11883 0 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
11884 :
11885 0 : if (iv_limit == -1)
11886 : return true;
11887 :
11888 0 : tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
11889 0 : unsigned int compare_precision = TYPE_PRECISION (compare_type);
11890 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
11891 :
11892 0 : if (wi::min_precision (iv_limit * nitems, UNSIGNED) > compare_precision)
11893 0 : return true;
11894 :
11895 : return false;
11896 0 : }
|
