Line data Source code
1 : /* Loop Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 : Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #define INCLUDE_ALGORITHM
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "cfghooks.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "optabs-tree.h"
35 : #include "memmodel.h"
36 : #include "optabs.h"
37 : #include "diagnostic-core.h"
38 : #include "fold-const.h"
39 : #include "stor-layout.h"
40 : #include "cfganal.h"
41 : #include "gimplify.h"
42 : #include "gimple-iterator.h"
43 : #include "gimplify-me.h"
44 : #include "tree-ssa-loop-ivopts.h"
45 : #include "tree-ssa-loop-manip.h"
46 : #include "tree-ssa-loop-niter.h"
47 : #include "tree-ssa-loop.h"
48 : #include "cfgloop.h"
49 : #include "tree-scalar-evolution.h"
50 : #include "tree-vectorizer.h"
51 : #include "gimple-fold.h"
52 : #include "cgraph.h"
53 : #include "tree-cfg.h"
54 : #include "tree-if-conv.h"
55 : #include "internal-fn.h"
56 : #include "tree-vector-builder.h"
57 : #include "vec-perm-indices.h"
58 : #include "tree-eh.h"
59 : #include "case-cfn-macros.h"
60 : #include "langhooks.h"
61 : #include "opts.h"
62 : #include "hierarchical_discriminator.h"
63 :
64 : /* Loop Vectorization Pass.
65 :
66 : This pass tries to vectorize loops.
67 :
68 : For example, the vectorizer transforms the following simple loop:
69 :
70 : short a[N]; short b[N]; short c[N]; int i;
71 :
72 : for (i=0; i<N; i++){
73 : a[i] = b[i] + c[i];
74 : }
75 :
76 : as if it was manually vectorized by rewriting the source code into:
77 :
78 : typedef int __attribute__((mode(V8HI))) v8hi;
79 : short a[N]; short b[N]; short c[N]; int i;
80 : v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
81 : v8hi va, vb, vc;
82 :
83 : for (i=0; i<N/8; i++){
84 : vb = pb[i];
85 : vc = pc[i];
86 : va = vb + vc;
87 : pa[i] = va;
88 : }
89 :
90 : The main entry to this pass is vectorize_loops(), in which
91 : the vectorizer applies a set of analyses on a given set of loops,
92 : followed by the actual vectorization transformation for the loops that
93 : had successfully passed the analysis phase.
94 : Throughout this pass we make a distinction between two types of
95 : data: scalars (which are represented by SSA_NAMES), and memory references
96 : ("data-refs"). These two types of data require different handling both
97 : during analysis and transformation. The types of data-refs that the
98 : vectorizer currently supports are ARRAY_REFS which base is an array DECL
99 : (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
100 : accesses are required to have a simple (consecutive) access pattern.
101 :
102 : Analysis phase:
103 : ===============
104 : The driver for the analysis phase is vect_analyze_loop().
105 : It applies a set of analyses, some of which rely on the scalar evolution
106 : analyzer (scev) developed by Sebastian Pop.
107 :
108 : During the analysis phase the vectorizer records some information
109 : per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
110 : loop, as well as general information about the loop as a whole, which is
111 : recorded in a "loop_vec_info" struct attached to each loop.
112 :
113 : Transformation phase:
114 : =====================
115 : The loop transformation phase scans all the stmts in the loop, and
116 : creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
117 : the loop that needs to be vectorized. It inserts the vector code sequence
118 : just before the scalar stmt S, and records a pointer to the vector code
119 : in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
120 : attached to S). This pointer will be used for the vectorization of following
121 : stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
122 : otherwise, we rely on dead code elimination for removing it.
123 :
124 : For example, say stmt S1 was vectorized into stmt VS1:
125 :
126 : VS1: vb = px[i];
127 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
128 : S2: a = b;
129 :
130 : To vectorize stmt S2, the vectorizer first finds the stmt that defines
131 : the operand 'b' (S1), and gets the relevant vector def 'vb' from the
132 : vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
133 : resulting sequence would be:
134 :
135 : VS1: vb = px[i];
136 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
137 : VS2: va = vb;
138 : S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
139 :
140 : Operands that are not SSA_NAMEs, are data-refs that appear in
141 : load/store operations (like 'x[i]' in S1), and are handled differently.
142 :
143 : Target modeling:
144 : =================
145 : Currently the only target specific information that is used is the
146 : size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
147 : Targets that can support different sizes of vectors, for now will need
148 : to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
149 : flexibility will be added in the future.
150 :
151 : Since we only vectorize operations which vector form can be
152 : expressed using existing tree codes, to verify that an operation is
153 : supported, the vectorizer checks the relevant optab at the relevant
154 : machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
155 : the value found is CODE_FOR_nothing, then there's no target support, and
156 : we can't vectorize the stmt.
157 :
158 : For additional information on this project see:
159 : http://gcc.gnu.org/projects/tree-ssa/vectorization.html
160 : */
161 :
162 : static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *,
163 : unsigned *);
164 : static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
165 : gphi **);
166 :
167 :
168 : /* Function vect_is_simple_iv_evolution.
169 :
170 : FORNOW: A simple evolution of an induction variables in the loop is
171 : considered a polynomial evolution. */
172 :
173 : static bool
174 957179 : vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn,
175 : stmt_vec_info stmt_info)
176 : {
177 957179 : tree init_expr;
178 957179 : tree step_expr;
179 957179 : tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
180 957179 : basic_block bb;
181 :
182 : /* When there is no evolution in this loop, the evolution function
183 : is not "simple". */
184 957179 : if (evolution_part == NULL_TREE)
185 : return false;
186 :
187 : /* When the evolution is a polynomial of degree >= 2
188 : the evolution function is not "simple". */
189 840903 : if (tree_is_chrec (evolution_part))
190 : return false;
191 :
192 840903 : step_expr = evolution_part;
193 840903 : init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
194 :
195 840903 : if (dump_enabled_p ())
196 41890 : dump_printf_loc (MSG_NOTE, vect_location, "step: %T, init: %T\n",
197 : step_expr, init_expr);
198 :
199 840903 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
200 840903 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step_expr;
201 840903 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_add;
202 :
203 840903 : if (TREE_CODE (step_expr) != INTEGER_CST
204 72696 : && (TREE_CODE (step_expr) != SSA_NAME
205 57548 : || ((bb = gimple_bb (SSA_NAME_DEF_STMT (step_expr)))
206 57292 : && flow_bb_inside_loop_p (get_loop (cfun, loop_nb), bb))
207 7779 : || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
208 133 : && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))
209 133 : || !flag_associative_math)))
210 905917 : && (TREE_CODE (step_expr) != REAL_CST
211 416 : || !flag_associative_math))
212 : {
213 64924 : if (dump_enabled_p ())
214 3230 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
215 : "step unknown.\n");
216 : return false;
217 : }
218 :
219 : return true;
220 : }
221 :
222 : /* Function vect_is_nonlinear_iv_evolution
223 :
224 : Only support nonlinear induction for integer type
225 : 1. neg
226 : 2. mul by constant
227 : 3. lshift/rshift by constant.
228 :
229 : For neg induction, return a fake step as integer -1. */
230 : static bool
231 178745 : vect_is_nonlinear_iv_evolution (class loop* loop, stmt_vec_info stmt_info,
232 : gphi* loop_phi_node)
233 : {
234 178745 : tree init_expr, ev_expr, result, op1, op2;
235 178745 : gimple* def;
236 :
237 178745 : if (gimple_phi_num_args (loop_phi_node) != 2)
238 : return false;
239 :
240 178745 : init_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_preheader_edge (loop));
241 178745 : ev_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_latch_edge (loop));
242 :
243 : /* Support nonlinear induction only for integer type. */
244 178745 : if (!INTEGRAL_TYPE_P (TREE_TYPE (init_expr)))
245 : return false;
246 :
247 111441 : result = PHI_RESULT (loop_phi_node);
248 :
249 111441 : if (TREE_CODE (ev_expr) != SSA_NAME
250 109000 : || ((def = SSA_NAME_DEF_STMT (ev_expr)), false)
251 111441 : || !is_gimple_assign (def))
252 : return false;
253 :
254 100518 : enum tree_code t_code = gimple_assign_rhs_code (def);
255 100518 : tree step;
256 100518 : switch (t_code)
257 : {
258 3525 : case NEGATE_EXPR:
259 3525 : if (gimple_assign_rhs1 (def) != result)
260 : return false;
261 3525 : step = build_int_cst (TREE_TYPE (init_expr), -1);
262 3525 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_neg;
263 3525 : break;
264 :
265 12243 : case RSHIFT_EXPR:
266 12243 : case LSHIFT_EXPR:
267 12243 : case MULT_EXPR:
268 12243 : op1 = gimple_assign_rhs1 (def);
269 12243 : op2 = gimple_assign_rhs2 (def);
270 12243 : if (TREE_CODE (op2) != INTEGER_CST
271 8255 : || op1 != result)
272 : return false;
273 7846 : step = op2;
274 7846 : if (t_code == LSHIFT_EXPR)
275 536 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shl;
276 7310 : else if (t_code == RSHIFT_EXPR)
277 6315 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shr;
278 : /* NEGATE_EXPR and MULT_EXPR are both vect_step_op_mul. */
279 : else
280 995 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_mul;
281 : break;
282 :
283 : default:
284 : return false;
285 : }
286 :
287 11371 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
288 11371 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step;
289 :
290 11371 : return true;
291 : }
292 :
293 : /* Returns true if Phi is a first-order recurrence. A first-order
294 : recurrence is a non-reduction recurrence relation in which the value of
295 : the recurrence in the current loop iteration equals a value defined in
296 : the previous iteration. */
297 :
298 : static bool
299 71122 : vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo, class loop *loop,
300 : gphi *phi)
301 : {
302 : /* A nested cycle isn't vectorizable as first order recurrence. */
303 71122 : if (LOOP_VINFO_LOOP (loop_vinfo) != loop)
304 : return false;
305 :
306 : /* Ensure the loop latch definition is from within the loop. */
307 70977 : edge latch = loop_latch_edge (loop);
308 70977 : tree ldef = PHI_ARG_DEF_FROM_EDGE (phi, latch);
309 70977 : if (TREE_CODE (ldef) != SSA_NAME
310 68199 : || SSA_NAME_IS_DEFAULT_DEF (ldef)
311 68133 : || is_a <gphi *> (SSA_NAME_DEF_STMT (ldef))
312 132886 : || !flow_bb_inside_loop_p (loop, gimple_bb (SSA_NAME_DEF_STMT (ldef))))
313 : return false;
314 :
315 61238 : tree def = gimple_phi_result (phi);
316 :
317 : /* Ensure every use_stmt of the phi node is dominated by the latch
318 : definition. */
319 61238 : imm_use_iterator imm_iter;
320 61238 : use_operand_p use_p;
321 74519 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, def)
322 73880 : if (!is_gimple_debug (USE_STMT (use_p))
323 143967 : && (SSA_NAME_DEF_STMT (ldef) == USE_STMT (use_p)
324 48480 : || !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (ldef),
325 : USE_STMT (use_p))))
326 60599 : return false;
327 :
328 : /* First-order recurrence autovectorization needs shuffle vector. */
329 639 : tree scalar_type = TREE_TYPE (def);
330 639 : tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
331 639 : if (!vectype)
332 6 : return false;
333 :
334 : return true;
335 : }
336 :
337 : /* Function vect_analyze_scalar_cycles_1.
338 :
339 : Examine the cross iteration def-use cycles of scalar variables
340 : in LOOP. LOOP_VINFO represents the loop that is now being
341 : considered for vectorization (can be LOOP, or an outer-loop
342 : enclosing LOOP). SLP indicates there will be some subsequent
343 : slp analyses or not. */
344 :
345 : static void
346 470784 : vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
347 : {
348 470784 : basic_block bb = loop->header;
349 470784 : auto_vec<stmt_vec_info, 64> worklist;
350 470784 : gphi_iterator gsi;
351 :
352 470784 : DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
353 :
354 : /* First - identify all inductions. Reduction detection assumes that all the
355 : inductions have been identified, therefore, this order must not be
356 : changed. */
357 1678730 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
358 : {
359 1207946 : gphi *phi = gsi.phi ();
360 1207946 : tree access_fn = NULL;
361 1207946 : tree def = PHI_RESULT (phi);
362 1207946 : stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi);
363 :
364 : /* Skip virtual phi's. The data dependences that are associated with
365 : virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
366 2415892 : if (virtual_operand_p (def))
367 420602 : continue;
368 :
369 : /* Skip already analyzed inner loop PHIs of double reductions. */
370 958206 : if (VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
371 1027 : continue;
372 :
373 957179 : if (dump_enabled_p ())
374 44157 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
375 : (gimple *) phi);
376 :
377 957179 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
378 :
379 : /* Analyze the evolution function. */
380 957179 : access_fn = analyze_scalar_evolution (loop, def);
381 957179 : if (dump_enabled_p ())
382 44157 : dump_printf_loc (MSG_NOTE, vect_location,
383 : "Access function of PHI: %T\n", access_fn);
384 957179 : if (access_fn)
385 957179 : STRIP_NOPS (access_fn);
386 :
387 1127014 : if ((!access_fn
388 957179 : || !vect_is_simple_iv_evolution (loop->num, access_fn, stmt_vinfo)
389 775979 : || (LOOP_VINFO_LOOP (loop_vinfo) != loop
390 11609 : && (TREE_CODE (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo))
391 : != INTEGER_CST)))
392 : /* Only handle nonlinear iv for same loop. */
393 1138385 : && (LOOP_VINFO_LOOP (loop_vinfo) != loop
394 178745 : || !vect_is_nonlinear_iv_evolution (loop, stmt_vinfo, phi)))
395 : {
396 169835 : worklist.safe_push (stmt_vinfo);
397 169835 : continue;
398 : }
399 :
400 787344 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo)
401 : != NULL_TREE);
402 787344 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo) != NULL_TREE);
403 :
404 787344 : if (dump_enabled_p ())
405 38882 : dump_printf_loc (MSG_NOTE, vect_location, "Detected induction.\n");
406 787344 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
407 :
408 : /* Mark if we have a non-linear IV. */
409 787344 : LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
410 787344 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add;
411 : }
412 :
413 :
414 : /* Second - identify all reductions and nested cycles. */
415 640619 : while (worklist.length () > 0)
416 : {
417 169835 : stmt_vec_info stmt_vinfo = worklist.pop ();
418 169835 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
419 169835 : tree def = PHI_RESULT (phi);
420 :
421 169835 : if (dump_enabled_p ())
422 5275 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
423 : (gimple *) phi);
424 :
425 339670 : gcc_assert (!virtual_operand_p (def)
426 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
427 :
428 169835 : gphi *double_reduc;
429 169835 : stmt_vec_info reduc_stmt_info
430 169835 : = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc);
431 169835 : if (reduc_stmt_info && double_reduc)
432 : {
433 1129 : stmt_vec_info inner_phi_info
434 1129 : = loop_vinfo->lookup_stmt (double_reduc);
435 : /* ??? Pass down flag we're the inner loop of a double reduc. */
436 1129 : stmt_vec_info inner_reduc_info
437 1129 : = vect_is_simple_reduction (loop_vinfo, inner_phi_info, NULL);
438 1129 : if (inner_reduc_info)
439 : {
440 1027 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
441 1027 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
442 1027 : STMT_VINFO_REDUC_DEF (inner_phi_info) = inner_reduc_info;
443 1027 : STMT_VINFO_REDUC_DEF (inner_reduc_info) = inner_phi_info;
444 1027 : if (dump_enabled_p ())
445 130 : dump_printf_loc (MSG_NOTE, vect_location,
446 : "Detected double reduction.\n");
447 :
448 1027 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
449 1027 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_double_reduction_def;
450 1027 : STMT_VINFO_DEF_TYPE (inner_phi_info) = vect_nested_cycle;
451 : /* Make it accessible for SLP vectorization. */
452 1027 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
453 : }
454 102 : else if (dump_enabled_p ())
455 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
456 : "Unknown def-use cycle pattern.\n");
457 : }
458 168706 : else if (reduc_stmt_info)
459 : {
460 97584 : if (loop != LOOP_VINFO_LOOP (loop_vinfo))
461 : {
462 2316 : if (dump_enabled_p ())
463 437 : dump_printf_loc (MSG_NOTE, vect_location,
464 : "Detected vectorizable nested cycle.\n");
465 :
466 2316 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
467 : }
468 : else
469 : {
470 95268 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
471 95268 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
472 95268 : if (dump_enabled_p ())
473 4120 : dump_printf_loc (MSG_NOTE, vect_location,
474 : "Detected reduction.\n");
475 :
476 95268 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
477 95268 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def;
478 95268 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
479 : }
480 : }
481 71122 : else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
482 633 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
483 : else
484 70489 : if (dump_enabled_p ())
485 497 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
486 : "Unknown def-use cycle pattern.\n");
487 : }
488 470784 : }
489 :
490 :
491 : /* Function vect_analyze_scalar_cycles.
492 :
493 : Examine the cross iteration def-use cycles of scalar variables, by
494 : analyzing the loop-header PHIs of scalar variables. Classify each
495 : cycle as one of the following: invariant, induction, reduction, unknown.
496 : We do that for the loop represented by LOOP_VINFO, and also to its
497 : inner-loop, if exists.
498 : Examples for scalar cycles:
499 :
500 : Example1: reduction:
501 :
502 : loop1:
503 : for (i=0; i<N; i++)
504 : sum += a[i];
505 :
506 : Example2: induction:
507 :
508 : loop2:
509 : for (i=0; i<N; i++)
510 : a[i] = i; */
511 :
512 : static void
513 464870 : vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
514 : {
515 464870 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
516 :
517 464870 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
518 :
519 : /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
520 : Reductions in such inner-loop therefore have different properties than
521 : the reductions in the nest that gets vectorized:
522 : 1. When vectorized, they are executed in the same order as in the original
523 : scalar loop, so we can't change the order of computation when
524 : vectorizing them.
525 : 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
526 : current checks are too strict. */
527 :
528 464870 : if (loop->inner)
529 5914 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
530 464870 : }
531 :
532 : /* Function vect_get_loop_niters.
533 :
534 : Determine how many iterations the loop is executed and place it
535 : in NUMBER_OF_ITERATIONS. Place the number of latch iterations
536 : in NUMBER_OF_ITERATIONSM1. Place the condition under which the
537 : niter information holds in ASSUMPTIONS.
538 :
539 : Return the loop exit conditions. */
540 :
541 :
542 : static vec<gcond *>
543 290813 : vect_get_loop_niters (class loop *loop, const_edge main_exit, tree *assumptions,
544 : tree *number_of_iterations, tree *number_of_iterationsm1)
545 : {
546 290813 : auto_vec<edge> exits = get_loop_exit_edges (loop);
547 290813 : vec<gcond *> conds;
548 581626 : conds.create (exits.length ());
549 290813 : class tree_niter_desc niter_desc;
550 290813 : tree niter_assumptions, niter, may_be_zero;
551 :
552 290813 : *assumptions = boolean_true_node;
553 290813 : *number_of_iterationsm1 = chrec_dont_know;
554 290813 : *number_of_iterations = chrec_dont_know;
555 :
556 290813 : DUMP_VECT_SCOPE ("get_loop_niters");
557 :
558 290813 : if (exits.is_empty ())
559 0 : return conds;
560 :
561 290813 : if (dump_enabled_p ())
562 15199 : dump_printf_loc (MSG_NOTE, vect_location, "Loop has %d exits.\n",
563 : exits.length ());
564 :
565 290813 : edge exit;
566 290813 : unsigned int i;
567 709211 : FOR_EACH_VEC_ELT (exits, i, exit)
568 : {
569 418398 : gcond *cond = get_loop_exit_condition (exit);
570 418398 : if (cond)
571 418364 : conds.safe_push (cond);
572 :
573 418398 : if (dump_enabled_p ())
574 16485 : dump_printf_loc (MSG_NOTE, vect_location, "Analyzing exit %d...\n", i);
575 :
576 418398 : if (exit != main_exit)
577 187892 : continue;
578 :
579 290813 : may_be_zero = NULL_TREE;
580 290813 : if (!number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
581 290813 : || chrec_contains_undetermined (niter_desc.niter))
582 60307 : continue;
583 :
584 230506 : niter_assumptions = niter_desc.assumptions;
585 230506 : may_be_zero = niter_desc.may_be_zero;
586 230506 : niter = niter_desc.niter;
587 :
588 230506 : if (may_be_zero && integer_zerop (may_be_zero))
589 : may_be_zero = NULL_TREE;
590 :
591 9661 : if (may_be_zero)
592 : {
593 9661 : if (COMPARISON_CLASS_P (may_be_zero))
594 : {
595 : /* Try to combine may_be_zero with assumptions, this can simplify
596 : computation of niter expression. */
597 9661 : if (niter_assumptions && !integer_nonzerop (niter_assumptions))
598 968 : niter_assumptions = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
599 : niter_assumptions,
600 : fold_build1 (TRUTH_NOT_EXPR,
601 : boolean_type_node,
602 : may_be_zero));
603 : else
604 8693 : niter = fold_build3 (COND_EXPR, TREE_TYPE (niter), may_be_zero,
605 : build_int_cst (TREE_TYPE (niter), 0),
606 : rewrite_to_non_trapping_overflow (niter));
607 :
608 230506 : may_be_zero = NULL_TREE;
609 : }
610 0 : else if (integer_nonzerop (may_be_zero))
611 : {
612 0 : *number_of_iterationsm1 = build_int_cst (TREE_TYPE (niter), 0);
613 0 : *number_of_iterations = build_int_cst (TREE_TYPE (niter), 1);
614 0 : continue;
615 : }
616 : else
617 0 : continue;
618 : }
619 :
620 : /* Loop assumptions are based off the normal exit. */
621 230506 : *assumptions = niter_assumptions;
622 230506 : *number_of_iterationsm1 = niter;
623 :
624 : /* We want the number of loop header executions which is the number
625 : of latch executions plus one.
626 : ??? For UINT_MAX latch executions this number overflows to zero
627 : for loops like do { n++; } while (n != 0); */
628 230506 : if (niter && !chrec_contains_undetermined (niter))
629 : {
630 230506 : niter = fold_build2 (PLUS_EXPR, TREE_TYPE (niter),
631 : unshare_expr (niter),
632 : build_int_cst (TREE_TYPE (niter), 1));
633 230506 : if (TREE_CODE (niter) == INTEGER_CST
634 126756 : && TREE_CODE (*number_of_iterationsm1) != INTEGER_CST)
635 : {
636 : /* If we manage to fold niter + 1 into INTEGER_CST even when
637 : niter is some complex expression, ensure back
638 : *number_of_iterationsm1 is an INTEGER_CST as well. See
639 : PR113210. */
640 0 : *number_of_iterationsm1
641 0 : = fold_build2 (PLUS_EXPR, TREE_TYPE (niter), niter,
642 : build_minus_one_cst (TREE_TYPE (niter)));
643 : }
644 : }
645 230506 : *number_of_iterations = niter;
646 : }
647 :
648 290813 : if (dump_enabled_p ())
649 15199 : dump_printf_loc (MSG_NOTE, vect_location, "All loop exits successfully analyzed.\n");
650 :
651 290813 : return conds;
652 290813 : }
653 :
654 : /* Determine the main loop exit for the vectorizer. */
655 :
656 : edge
657 503334 : vec_init_loop_exit_info (class loop *loop)
658 : {
659 : /* Before we begin we must first determine which exit is the main one and
660 : which are auxiliary exits. */
661 503334 : auto_vec<edge> exits = get_loop_exit_edges (loop);
662 1001575 : if (exits.length () == 0)
663 : return NULL;
664 498241 : if (exits.length () == 1)
665 325661 : return exits[0];
666 :
667 : /* If we have multiple exits, look for counting IV exit.
668 : Analyze all exits and return the last one we can analyze. */
669 172580 : class tree_niter_desc niter_desc;
670 172580 : edge candidate = NULL;
671 637727 : for (edge exit : exits)
672 : {
673 486412 : if (!get_loop_exit_condition (exit))
674 : {
675 21265 : if (dump_enabled_p ())
676 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
677 : "Unhandled loop exit detected.\n");
678 : return NULL;
679 : }
680 :
681 465147 : if (number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
682 465147 : && !chrec_contains_undetermined (niter_desc.niter))
683 : {
684 138293 : tree may_be_zero = niter_desc.may_be_zero;
685 138293 : if ((integer_zerop (may_be_zero)
686 : /* As we are handling may_be_zero that's not false by
687 : rewriting niter to may_be_zero ? 0 : niter we require
688 : an empty latch. */
689 475646 : || (single_pred_p (loop->latch)
690 10298 : && exit->src == single_pred (loop->latch)
691 2555 : && (integer_nonzerop (may_be_zero)
692 2555 : || COMPARISON_CLASS_P (may_be_zero))))
693 140848 : && (!candidate
694 5842 : || dominated_by_p (CDI_DOMINATORS, exit->src,
695 5842 : candidate->src)))
696 : candidate = exit;
697 : }
698 : }
699 :
700 : /* If no exit is analyzable by scalar evolution, we return the last exit
701 : under the assummption we are dealing with an uncounted loop. */
702 208225 : if (!candidate && single_pred_p (loop->latch))
703 35645 : candidate = loop_exits_from_bb_p (loop, single_pred (loop->latch));
704 :
705 : return candidate;
706 172580 : }
707 :
708 : /* Function bb_in_loop_p
709 :
710 : Used as predicate for dfs order traversal of the loop bbs. */
711 :
712 : static bool
713 2485886 : bb_in_loop_p (const_basic_block bb, const void *data)
714 : {
715 2485886 : const class loop *const loop = (const class loop *)data;
716 2485886 : if (flow_bb_inside_loop_p (loop, bb))
717 : return true;
718 : return false;
719 : }
720 :
721 : /* Get and return the basic blocks of LOOP in DFS order. Store the number of
722 : BBs in NBBS. */
723 :
724 : static basic_block *
725 440546 : vect_get_loop_dfs_bbs (class loop *loop, unsigned int *nbbs)
726 : {
727 : /* CHECKME: We want to visit all BBs before their successors (except for
728 : latch blocks, for which this assertion wouldn't hold). In the simple
729 : case of the loop forms we allow, a dfs order of the BBs would the same
730 : as reversed postorder traversal, so we are safe. */
731 440546 : basic_block *bbs = XCNEWVEC (basic_block, loop->num_nodes);
732 881092 : *nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p, bbs,
733 440546 : loop->num_nodes, loop);
734 440546 : gcc_assert (*nbbs == loop->num_nodes);
735 440546 : return bbs;
736 : }
737 :
738 : /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
739 : stmt_vec_info structs for all the stmts in LOOP_IN. */
740 :
741 541421 : _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared,
742 : basic_block *bbs_in)
743 : : vec_info (vec_info::loop, shared),
744 541421 : loop (loop_in),
745 541421 : num_itersm1 (NULL_TREE),
746 541421 : num_iters (NULL_TREE),
747 541421 : num_iters_unchanged (NULL_TREE),
748 541421 : num_iters_assumptions (NULL_TREE),
749 541421 : vector_costs (nullptr),
750 541421 : scalar_costs (nullptr),
751 541421 : th (0),
752 541421 : versioning_threshold (0),
753 541421 : vectorization_factor (0),
754 541421 : main_loop_edge (nullptr),
755 541421 : skip_main_loop_edge (nullptr),
756 541421 : skip_this_loop_edge (nullptr),
757 541421 : reusable_accumulators (),
758 541421 : suggested_unroll_factor (1),
759 541421 : max_vectorization_factor (0),
760 541421 : mask_skip_niters (NULL_TREE),
761 541421 : mask_skip_niters_pfa_offset (NULL_TREE),
762 541421 : rgroup_compare_type (NULL_TREE),
763 541421 : simd_if_cond (NULL_TREE),
764 541421 : partial_vector_style (vect_partial_vectors_none),
765 541421 : unaligned_dr (NULL),
766 541421 : peeling_for_alignment (0),
767 541421 : ptr_mask (0),
768 541421 : max_spec_read_amount (0),
769 541421 : nonlinear_iv (false),
770 541421 : ivexpr_map (NULL),
771 541421 : scan_map (NULL),
772 541421 : inner_loop_cost_factor (param_vect_inner_loop_cost_factor),
773 541421 : vectorizable (false),
774 541421 : can_use_partial_vectors_p (true),
775 541421 : must_use_partial_vectors_p (false),
776 541421 : using_partial_vectors_p (false),
777 541421 : using_decrementing_iv_p (false),
778 541421 : using_select_vl_p (false),
779 541421 : allow_mutual_alignment (false),
780 541421 : partial_load_store_bias (0),
781 541421 : peeling_for_gaps (false),
782 541421 : peeling_for_niter (false),
783 541421 : early_breaks (false),
784 541421 : loop_iv_cond (NULL),
785 541421 : user_unroll (false),
786 541421 : no_data_dependencies (false),
787 541421 : has_mask_store (false),
788 541421 : scalar_loop_scaling (profile_probability::uninitialized ()),
789 541421 : scalar_loop (NULL),
790 541421 : main_loop_info (NULL),
791 541421 : orig_loop_info (NULL),
792 541421 : epilogue_vinfo (NULL),
793 541421 : drs_advanced_by (NULL_TREE),
794 541421 : vec_loop_main_exit (NULL),
795 541421 : vec_epilogue_loop_main_exit (NULL),
796 541421 : scalar_loop_main_exit (NULL),
797 541421 : early_break_needs_epilogue (false),
798 541421 : early_break_niters_var (NULL)
799 : {
800 541421 : nbbs = loop->num_nodes;
801 541421 : bbs = XNEWVEC (basic_block, nbbs);
802 541421 : memcpy (bbs, bbs_in, nbbs * sizeof (basic_block));
803 :
804 1884413 : for (unsigned int i = 0; i < nbbs; i++)
805 : {
806 1342992 : basic_block bb = bbs[i];
807 1342992 : gimple_stmt_iterator si;
808 :
809 2768344 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
810 : {
811 1425352 : gimple *phi = gsi_stmt (si);
812 1425352 : gimple_set_uid (phi, 0);
813 1425352 : add_stmt (phi);
814 : }
815 :
816 12383729 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
817 : {
818 9697745 : gimple *stmt = gsi_stmt (si);
819 9697745 : gimple_set_uid (stmt, 0);
820 9697745 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
821 4122416 : continue;
822 5575329 : add_stmt (stmt);
823 : /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
824 : third argument is the #pragma omp simd if (x) condition, when 0,
825 : loop shouldn't be vectorized, when non-zero constant, it should
826 : be vectorized normally, otherwise versioned with vectorized loop
827 : done if the condition is non-zero at runtime. */
828 5575329 : if (loop_in->simduid
829 43184 : && is_gimple_call (stmt)
830 4245 : && gimple_call_internal_p (stmt)
831 4123 : && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
832 4122 : && gimple_call_num_args (stmt) >= 3
833 104 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
834 5575433 : && (loop_in->simduid
835 104 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0))))
836 : {
837 104 : tree arg = gimple_call_arg (stmt, 2);
838 104 : if (integer_zerop (arg) || TREE_CODE (arg) == SSA_NAME)
839 104 : simd_if_cond = arg;
840 : else
841 0 : gcc_assert (integer_nonzerop (arg));
842 : }
843 : }
844 : }
845 541421 : }
846 :
847 : /* Free all levels of rgroup CONTROLS. */
848 :
849 : void
850 1391272 : release_vec_loop_controls (vec<rgroup_controls> *controls)
851 : {
852 1391272 : rgroup_controls *rgc;
853 1391272 : unsigned int i;
854 1416497 : FOR_EACH_VEC_ELT (*controls, i, rgc)
855 25225 : rgc->controls.release ();
856 1391272 : controls->release ();
857 1391272 : }
858 :
859 : /* Free all memory used by the _loop_vec_info, as well as all the
860 : stmt_vec_info structs of all the stmts in the loop. */
861 :
862 541421 : _loop_vec_info::~_loop_vec_info ()
863 : {
864 541421 : free (bbs);
865 :
866 541421 : release_vec_loop_controls (&masks.rgc_vec);
867 541421 : release_vec_loop_controls (&lens);
868 545486 : delete ivexpr_map;
869 541743 : delete scan_map;
870 541421 : delete scalar_costs;
871 541421 : delete vector_costs;
872 757891 : for (auto reduc_info : reduc_infos)
873 204364 : delete reduc_info;
874 :
875 : /* When we release an epiloge vinfo that we do not intend to use
876 : avoid clearing AUX of the main loop which should continue to
877 : point to the main loop vinfo since otherwise we'll leak that. */
878 541421 : if (loop->aux == this)
879 64396 : loop->aux = NULL;
880 1082842 : }
881 :
882 : /* Return an invariant or register for EXPR and emit necessary
883 : computations in the LOOP_VINFO loop preheader. */
884 :
885 : tree
886 21048 : cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo, tree expr)
887 : {
888 21048 : if (is_gimple_reg (expr)
889 21048 : || is_gimple_min_invariant (expr))
890 : return expr;
891 :
892 13731 : if (! loop_vinfo->ivexpr_map)
893 4065 : loop_vinfo->ivexpr_map = new hash_map<tree_operand_hash, tree>;
894 13731 : tree &cached = loop_vinfo->ivexpr_map->get_or_insert (expr);
895 13731 : if (! cached)
896 : {
897 8880 : gimple_seq stmts = NULL;
898 8880 : cached = force_gimple_operand (unshare_expr (expr),
899 : &stmts, true, NULL_TREE);
900 8880 : if (stmts)
901 : {
902 8726 : edge e = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
903 8726 : gsi_insert_seq_on_edge_immediate (e, stmts);
904 : }
905 : }
906 13731 : return cached;
907 : }
908 :
909 : /* Return true if we can use CMP_TYPE as the comparison type to produce
910 : all masks required to mask LOOP_VINFO. */
911 :
912 : static bool
913 113306 : can_produce_all_loop_masks_p (loop_vec_info loop_vinfo, tree cmp_type)
914 : {
915 113306 : rgroup_controls *rgm;
916 113306 : unsigned int i;
917 129063 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
918 129063 : if (rgm->type != NULL_TREE
919 129063 : && !direct_internal_fn_supported_p (IFN_WHILE_ULT,
920 : cmp_type, rgm->type,
921 : OPTIMIZE_FOR_SPEED))
922 : return false;
923 : return true;
924 : }
925 :
926 : /* Calculate the maximum number of scalars per iteration for every
927 : rgroup in LOOP_VINFO. */
928 :
929 : static unsigned int
930 24132 : vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo)
931 : {
932 24132 : unsigned int res = 1;
933 24132 : unsigned int i;
934 24132 : rgroup_controls *rgm;
935 57650 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
936 33518 : res = MAX (res, rgm->max_nscalars_per_iter);
937 24132 : return res;
938 : }
939 :
940 : /* Calculate the minimum precision necessary to represent:
941 :
942 : MAX_NITERS * FACTOR
943 :
944 : as an unsigned integer, where MAX_NITERS is the maximum number of
945 : loop header iterations for the original scalar form of LOOP_VINFO. */
946 :
947 : unsigned
948 26668 : vect_min_prec_for_max_niters (loop_vec_info loop_vinfo, unsigned int factor)
949 : {
950 26668 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
951 :
952 : /* Get the maximum number of iterations that is representable
953 : in the counter type. */
954 26668 : tree ni_type;
955 26668 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
956 26668 : ni_type = TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo));
957 : else
958 0 : ni_type = sizetype;
959 26668 : widest_int max_ni = wi::to_widest (TYPE_MAX_VALUE (ni_type)) + 1;
960 :
961 : /* Get a more refined estimate for the number of iterations. */
962 26668 : widest_int max_back_edges;
963 26668 : if (max_loop_iterations (loop, &max_back_edges))
964 26668 : max_ni = wi::smin (max_ni, max_back_edges + 1);
965 :
966 : /* Work out how many bits we need to represent the limit. */
967 26668 : return wi::min_precision (max_ni * factor, UNSIGNED);
968 26668 : }
969 :
970 : /* True if the loop needs peeling or partial vectors when vectorized. */
971 :
972 : static bool
973 160574 : vect_need_peeling_or_partial_vectors_p (loop_vec_info loop_vinfo)
974 : {
975 160574 : unsigned HOST_WIDE_INT const_vf;
976 :
977 160574 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
978 : return true;
979 :
980 13971 : loop_vec_info main_loop_vinfo
981 159259 : = (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
982 159259 : ? LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) : loop_vinfo);
983 159259 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
984 81825 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo) >= 0)
985 : {
986 : /* Work out the (constant) number of iterations that need to be
987 : peeled for reasons other than niters. */
988 81774 : unsigned int peel_niter
989 : = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
990 81774 : return !multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter,
991 81774 : LOOP_VINFO_VECT_FACTOR (loop_vinfo));
992 : }
993 :
994 77485 : if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo)
995 77485 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&const_vf))
996 : {
997 : /* When the number of iterations is a multiple of the vectorization
998 : factor and we are not doing prologue or forced epilogue peeling
999 : the epilogue isn't necessary. */
1000 77058 : if (tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
1001 154116 : >= (unsigned) exact_log2 (const_vf))
1002 1802 : return false;
1003 : }
1004 :
1005 : return true;
1006 : }
1007 :
1008 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
1009 : whether we can actually generate the masks required. Return true if so,
1010 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_COMPARE_TYPE. */
1011 :
1012 : static bool
1013 24132 : vect_verify_full_masking (loop_vec_info loop_vinfo)
1014 : {
1015 24132 : unsigned int min_ni_width;
1016 :
1017 : /* Use a normal loop if there are no statements that need masking.
1018 : This only happens in rare degenerate cases: it means that the loop
1019 : has no loads, no stores, and no live-out values. */
1020 24132 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1021 : return false;
1022 :
1023 : /* Produce the rgroup controls. */
1024 59421 : for (auto mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1025 : {
1026 35289 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1027 35289 : tree vectype = mask.first;
1028 35289 : unsigned nvectors = mask.second;
1029 :
1030 46446 : if (masks->rgc_vec.length () < nvectors)
1031 26328 : masks->rgc_vec.safe_grow_cleared (nvectors, true);
1032 35289 : rgroup_controls *rgm = &(*masks).rgc_vec[nvectors - 1];
1033 : /* The number of scalars per iteration and the number of vectors are
1034 : both compile-time constants. */
1035 35289 : unsigned int nscalars_per_iter
1036 35289 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1037 35289 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1038 :
1039 35289 : if (rgm->max_nscalars_per_iter < nscalars_per_iter)
1040 : {
1041 28206 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1042 28206 : rgm->type = truth_type_for (vectype);
1043 28206 : rgm->factor = 1;
1044 : }
1045 : }
1046 :
1047 24132 : unsigned int max_nscalars_per_iter
1048 24132 : = vect_get_max_nscalars_per_iter (loop_vinfo);
1049 :
1050 : /* Work out how many bits we need to represent the limit. */
1051 24132 : min_ni_width
1052 24132 : = vect_min_prec_for_max_niters (loop_vinfo, max_nscalars_per_iter);
1053 :
1054 : /* Find a scalar mode for which WHILE_ULT is supported. */
1055 24132 : opt_scalar_int_mode cmp_mode_iter;
1056 24132 : tree cmp_type = NULL_TREE;
1057 24132 : tree iv_type = NULL_TREE;
1058 24132 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1059 24132 : unsigned int iv_precision = UINT_MAX;
1060 :
1061 24132 : if (iv_limit != -1)
1062 24132 : iv_precision = wi::min_precision (iv_limit * max_nscalars_per_iter,
1063 : UNSIGNED);
1064 :
1065 193056 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1066 : {
1067 168924 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1068 168924 : if (cmp_bits >= min_ni_width
1069 168924 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1070 : {
1071 113306 : tree this_type = build_nonstandard_integer_type (cmp_bits, true);
1072 113306 : if (this_type
1073 113306 : && can_produce_all_loop_masks_p (loop_vinfo, this_type))
1074 : {
1075 : /* Although we could stop as soon as we find a valid mode,
1076 : there are at least two reasons why that's not always the
1077 : best choice:
1078 :
1079 : - An IV that's Pmode or wider is more likely to be reusable
1080 : in address calculations than an IV that's narrower than
1081 : Pmode.
1082 :
1083 : - Doing the comparison in IV_PRECISION or wider allows
1084 : a natural 0-based IV, whereas using a narrower comparison
1085 : type requires mitigations against wrap-around.
1086 :
1087 : Conversely, if the IV limit is variable, doing the comparison
1088 : in a wider type than the original type can introduce
1089 : unnecessary extensions, so picking the widest valid mode
1090 : is not always a good choice either.
1091 :
1092 : Here we prefer the first IV type that's Pmode or wider,
1093 : and the first comparison type that's IV_PRECISION or wider.
1094 : (The comparison type must be no wider than the IV type,
1095 : to avoid extensions in the vector loop.)
1096 :
1097 : ??? We might want to try continuing beyond Pmode for ILP32
1098 : targets if CMP_BITS < IV_PRECISION. */
1099 0 : iv_type = this_type;
1100 0 : if (!cmp_type || iv_precision > TYPE_PRECISION (cmp_type))
1101 : cmp_type = this_type;
1102 0 : if (cmp_bits >= GET_MODE_BITSIZE (Pmode))
1103 : break;
1104 : }
1105 : }
1106 : }
1107 :
1108 24132 : if (!cmp_type)
1109 : {
1110 24132 : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.release ();
1111 24132 : return false;
1112 : }
1113 :
1114 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
1115 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1116 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_while_ult;
1117 0 : return true;
1118 24132 : }
1119 :
1120 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
1121 : whether we can actually generate AVX512 style masks. Return true if so,
1122 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_IV_TYPE. */
1123 :
1124 : static bool
1125 24132 : vect_verify_full_masking_avx512 (loop_vec_info loop_vinfo)
1126 : {
1127 : /* Produce differently organized rgc_vec and differently check
1128 : we can produce masks. */
1129 :
1130 : /* Use a normal loop if there are no statements that need masking.
1131 : This only happens in rare degenerate cases: it means that the loop
1132 : has no loads, no stores, and no live-out values. */
1133 24132 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1134 : return false;
1135 :
1136 : /* For the decrementing IV we need to represent all values in
1137 : [0, niter + niter_skip] where niter_skip is the elements we
1138 : skip in the first iteration for prologue peeling. */
1139 24132 : tree iv_type = NULL_TREE;
1140 24132 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1141 24132 : unsigned int iv_precision = UINT_MAX;
1142 24132 : if (iv_limit != -1)
1143 24132 : iv_precision = wi::min_precision (iv_limit, UNSIGNED);
1144 :
1145 : /* First compute the type for the IV we use to track the remaining
1146 : scalar iterations. */
1147 24132 : opt_scalar_int_mode cmp_mode_iter;
1148 31474 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1149 : {
1150 31474 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1151 31474 : if (cmp_bits >= iv_precision
1152 31474 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1153 : {
1154 24132 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
1155 24132 : if (iv_type)
1156 : break;
1157 : }
1158 : }
1159 24132 : if (!iv_type)
1160 : return false;
1161 :
1162 24132 : bool ok = true;
1163 :
1164 : /* Produce the rgroup controls. */
1165 51698 : for (auto const &mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1166 : {
1167 33720 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1168 33720 : tree vectype = mask.first;
1169 33720 : unsigned nvectors = mask.second;
1170 :
1171 : /* The number of scalars per iteration and the number of vectors are
1172 : both compile-time constants. */
1173 33720 : unsigned int nscalars_per_iter
1174 33720 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1175 33720 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1176 :
1177 : /* We index the rgroup_controls vector with nscalars_per_iter
1178 : which we keep constant and instead have a varying nvectors,
1179 : remembering the vector mask with the fewest nV. */
1180 43308 : if (masks->rgc_vec.length () < nscalars_per_iter)
1181 24165 : masks->rgc_vec.safe_grow_cleared (nscalars_per_iter, true);
1182 33720 : rgroup_controls *rgm = &(*masks).rgc_vec[nscalars_per_iter - 1];
1183 :
1184 : /* For now vect_get_loop_mask only supports integer mode masks
1185 : when we need to split it. */
1186 33720 : tree mask_type = truth_type_for (vectype);
1187 33720 : if (GET_MODE_CLASS (TYPE_MODE (mask_type)) != MODE_INT
1188 33720 : || TYPE_PRECISION (TREE_TYPE (mask_type)) != 1)
1189 : {
1190 6154 : ok = false;
1191 6154 : break;
1192 : }
1193 :
1194 27566 : if (!rgm->type || rgm->factor > nvectors)
1195 : {
1196 19701 : rgm->type = mask_type;
1197 19701 : rgm->compare_type = NULL_TREE;
1198 19701 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1199 19701 : rgm->factor = nvectors;
1200 19701 : rgm->bias_adjusted_ctrl = NULL_TREE;
1201 : }
1202 : }
1203 24132 : if (!ok)
1204 : {
1205 6154 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
1206 6154 : return false;
1207 : }
1208 :
1209 : /* There is no fixed compare type we are going to use but we have to
1210 : be able to get at one for each mask group. */
1211 17978 : unsigned int min_ni_width
1212 17978 : = wi::min_precision (vect_max_vf (loop_vinfo), UNSIGNED);
1213 :
1214 72507 : for (auto &rgc : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
1215 : {
1216 18577 : tree mask_type = rgc.type;
1217 18577 : if (!mask_type)
1218 520 : continue;
1219 :
1220 : /* If iv_type is usable as compare type use that - we can elide the
1221 : saturation in that case. */
1222 18057 : if (TYPE_PRECISION (iv_type) >= min_ni_width)
1223 : {
1224 18057 : tree cmp_vectype
1225 18057 : = build_vector_type (iv_type, TYPE_VECTOR_SUBPARTS (mask_type));
1226 18057 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1227 6032 : rgc.compare_type = cmp_vectype;
1228 : }
1229 18057 : if (!rgc.compare_type)
1230 34554 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1231 : {
1232 34550 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1233 34550 : if (cmp_bits >= min_ni_width
1234 34550 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1235 : {
1236 34538 : tree cmp_type = build_nonstandard_integer_type (cmp_bits, true);
1237 34538 : if (!cmp_type)
1238 0 : continue;
1239 :
1240 : /* Check whether we can produce the mask with cmp_type. */
1241 34538 : tree cmp_vectype
1242 34538 : = build_vector_type (cmp_type, TYPE_VECTOR_SUBPARTS (mask_type));
1243 34538 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1244 : {
1245 12021 : rgc.compare_type = cmp_vectype;
1246 12021 : break;
1247 : }
1248 : }
1249 : }
1250 18057 : if (!rgc.compare_type)
1251 : {
1252 : ok = false;
1253 : break;
1254 : }
1255 : }
1256 17978 : if (!ok)
1257 : {
1258 4 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
1259 4 : return false;
1260 : }
1261 :
1262 17974 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = error_mark_node;
1263 17974 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1264 17974 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_avx512;
1265 17974 : return true;
1266 24132 : }
1267 :
1268 : /* Check whether we can use vector access with length based on precision
1269 : comparison. So far, to keep it simple, we only allow the case that the
1270 : precision of the target supported length is larger than the precision
1271 : required by loop niters. */
1272 :
1273 : static bool
1274 6 : vect_verify_loop_lens (loop_vec_info loop_vinfo)
1275 : {
1276 6 : if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
1277 : return false;
1278 :
1279 0 : if (!VECTOR_MODE_P (loop_vinfo->vector_mode))
1280 : return false;
1281 :
1282 0 : machine_mode len_load_mode, len_store_mode;
1283 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
1284 0 : .exists (&len_load_mode))
1285 0 : return false;
1286 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
1287 0 : .exists (&len_store_mode))
1288 0 : return false;
1289 :
1290 0 : signed char partial_load_bias = internal_len_load_store_bias
1291 0 : (IFN_LEN_LOAD, len_load_mode);
1292 :
1293 0 : signed char partial_store_bias = internal_len_load_store_bias
1294 0 : (IFN_LEN_STORE, len_store_mode);
1295 :
1296 0 : gcc_assert (partial_load_bias == partial_store_bias);
1297 :
1298 0 : if (partial_load_bias == VECT_PARTIAL_BIAS_UNSUPPORTED)
1299 : return false;
1300 :
1301 : /* If the backend requires a bias of -1 for LEN_LOAD, we must not emit
1302 : len_loads with a length of zero. In order to avoid that we prohibit
1303 : more than one loop length here. */
1304 0 : if (partial_load_bias == -1
1305 0 : && LOOP_VINFO_LENS (loop_vinfo).length () > 1)
1306 : return false;
1307 :
1308 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) = partial_load_bias;
1309 :
1310 0 : unsigned int max_nitems_per_iter = 1;
1311 0 : unsigned int i;
1312 0 : rgroup_controls *rgl;
1313 : /* Find the maximum number of items per iteration for every rgroup. */
1314 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), i, rgl)
1315 : {
1316 0 : unsigned nitems_per_iter = rgl->max_nscalars_per_iter * rgl->factor;
1317 0 : max_nitems_per_iter = MAX (max_nitems_per_iter, nitems_per_iter);
1318 : }
1319 :
1320 : /* Work out how many bits we need to represent the length limit. */
1321 0 : unsigned int min_ni_prec
1322 0 : = vect_min_prec_for_max_niters (loop_vinfo, max_nitems_per_iter);
1323 :
1324 : /* Now use the maximum of below precisions for one suitable IV type:
1325 : - the IV's natural precision
1326 : - the precision needed to hold: the maximum number of scalar
1327 : iterations multiplied by the scale factor (min_ni_prec above)
1328 : - the Pmode precision
1329 :
1330 : If min_ni_prec is less than the precision of the current niters,
1331 : we prefer to still use the niters type. Prefer to use Pmode and
1332 : wider IV to avoid narrow conversions. */
1333 :
1334 0 : unsigned int ni_prec
1335 0 : = TYPE_PRECISION (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)));
1336 0 : min_ni_prec = MAX (min_ni_prec, ni_prec);
1337 0 : min_ni_prec = MAX (min_ni_prec, GET_MODE_BITSIZE (Pmode));
1338 :
1339 0 : tree iv_type = NULL_TREE;
1340 0 : opt_scalar_int_mode tmode_iter;
1341 0 : FOR_EACH_MODE_IN_CLASS (tmode_iter, MODE_INT)
1342 : {
1343 0 : scalar_mode tmode = tmode_iter.require ();
1344 0 : unsigned int tbits = GET_MODE_BITSIZE (tmode);
1345 :
1346 : /* ??? Do we really want to construct one IV whose precision exceeds
1347 : BITS_PER_WORD? */
1348 0 : if (tbits > BITS_PER_WORD)
1349 : break;
1350 :
1351 : /* Find the first available standard integral type. */
1352 0 : if (tbits >= min_ni_prec && targetm.scalar_mode_supported_p (tmode))
1353 : {
1354 0 : iv_type = build_nonstandard_integer_type (tbits, true);
1355 0 : break;
1356 : }
1357 : }
1358 :
1359 0 : if (!iv_type)
1360 : {
1361 0 : if (dump_enabled_p ())
1362 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1363 : "can't vectorize with length-based partial vectors"
1364 : " because there is no suitable iv type.\n");
1365 : return false;
1366 : }
1367 :
1368 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = iv_type;
1369 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1370 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_len;
1371 :
1372 0 : return true;
1373 : }
1374 :
1375 : /* Calculate the cost of one scalar iteration of the loop. */
1376 : static void
1377 386642 : vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
1378 : {
1379 386642 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1380 386642 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
1381 386642 : int nbbs = loop->num_nodes, i;
1382 :
1383 386642 : DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
1384 :
1385 : /* Gather costs for statements in the scalar loop. */
1386 :
1387 1332258 : for (i = 0; i < nbbs; i++)
1388 : {
1389 945616 : gimple_stmt_iterator si;
1390 945616 : basic_block bb = bbs[i];
1391 :
1392 7827326 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
1393 : {
1394 5936094 : gimple *stmt = gsi_stmt (si);
1395 5936094 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
1396 :
1397 5936094 : if (!is_gimple_assign (stmt)
1398 : && !is_gimple_call (stmt)
1399 : && !is_a<gcond *> (stmt))
1400 2201921 : continue;
1401 :
1402 : /* Skip stmts that are not vectorized inside the loop. */
1403 3734173 : stmt_vec_info vstmt_info = vect_stmt_to_vectorize (stmt_info);
1404 3734173 : if (!STMT_VINFO_RELEVANT_P (vstmt_info)
1405 1844116 : && (!STMT_VINFO_LIVE_P (vstmt_info)
1406 51 : || !VECTORIZABLE_CYCLE_DEF
1407 : (STMT_VINFO_DEF_TYPE (vstmt_info))))
1408 1844116 : continue;
1409 :
1410 1890057 : vect_cost_for_stmt kind;
1411 1890057 : if (STMT_VINFO_DATA_REF (stmt_info))
1412 : {
1413 908209 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
1414 : kind = scalar_load;
1415 : else
1416 339141 : kind = scalar_store;
1417 : }
1418 981848 : else if (vect_nop_conversion_p (stmt_info))
1419 57632 : continue;
1420 : else
1421 : kind = scalar_stmt;
1422 :
1423 : /* The target cost model applies the inner-loop frequency. */
1424 1832425 : record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
1425 : 1, kind, stmt_info, 0, vect_body);
1426 : }
1427 : }
1428 :
1429 : /* Now accumulate cost. */
1430 386642 : loop_vinfo->scalar_costs = init_cost (loop_vinfo, true);
1431 386642 : add_stmt_costs (loop_vinfo->scalar_costs,
1432 : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo));
1433 386642 : loop_vinfo->scalar_costs->finish_cost (nullptr);
1434 386642 : }
1435 :
1436 : /* Function vect_analyze_loop_form.
1437 :
1438 : Verify that certain CFG restrictions hold, including:
1439 : - the loop has a pre-header
1440 : - the loop has a single entry
1441 : - nested loops can have only a single exit.
1442 : - the loop exit condition is simple enough
1443 : - the number of iterations can be analyzed, i.e, a countable loop. The
1444 : niter could be analyzed under some assumptions. */
1445 :
1446 : opt_result
1447 470810 : vect_analyze_loop_form (class loop *loop, gimple *loop_vectorized_call,
1448 : vect_loop_form_info *info)
1449 : {
1450 470810 : DUMP_VECT_SCOPE ("vect_analyze_loop_form");
1451 :
1452 470810 : edge exit_e = vec_init_loop_exit_info (loop);
1453 470810 : if (!exit_e)
1454 30264 : return opt_result::failure_at (vect_location,
1455 : "not vectorized:"
1456 : " Infinite loop detected.\n");
1457 440546 : if (loop_vectorized_call)
1458 : {
1459 25515 : tree arg = gimple_call_arg (loop_vectorized_call, 1);
1460 25515 : class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
1461 25515 : edge scalar_exit_e = vec_init_loop_exit_info (scalar_loop);
1462 25515 : if (!scalar_exit_e)
1463 0 : return opt_result::failure_at (vect_location,
1464 : "not vectorized:"
1465 : " could not determine main exit from"
1466 : " loop with multiple exits.\n");
1467 : }
1468 :
1469 440546 : info->loop_exit = exit_e;
1470 440546 : if (dump_enabled_p ())
1471 16618 : dump_printf_loc (MSG_NOTE, vect_location,
1472 : "using as main loop exit: %d -> %d [AUX: %p]\n",
1473 16618 : exit_e->src->index, exit_e->dest->index, exit_e->aux);
1474 :
1475 : /* Check if we have any control flow that doesn't leave the loop. */
1476 440546 : unsigned int nbbs;
1477 440546 : info->bbs = vect_get_loop_dfs_bbs (loop, &nbbs);
1478 1351095 : for (unsigned i = 0; i < nbbs; i++)
1479 1028170 : if (EDGE_COUNT (info->bbs[i]->succs) != 1
1480 1028170 : && (EDGE_COUNT (info->bbs[i]->succs) != 2
1481 644502 : || !loop_exits_from_bb_p (info->bbs[i]->loop_father,
1482 : info->bbs[i])))
1483 117621 : return opt_result::failure_at (vect_location,
1484 : "not vectorized:"
1485 : " unsupported control flow in loop.\n");
1486 :
1487 : /* Check if we have any control flow that doesn't leave the loop. */
1488 324023 : bool has_phi = false;
1489 324023 : for (unsigned i = 0; i < nbbs; i++)
1490 323558 : if (!gimple_seq_empty_p (phi_nodes (info->bbs[i])))
1491 : {
1492 : has_phi = true;
1493 : break;
1494 : }
1495 322925 : if (!has_phi)
1496 465 : return opt_result::failure_at (vect_location,
1497 : "not vectorized:"
1498 : " no scalar evolution detected in loop.\n");
1499 :
1500 : /* Different restrictions apply when we are considering an inner-most loop,
1501 : vs. an outer (nested) loop.
1502 : (FORNOW. May want to relax some of these restrictions in the future). */
1503 :
1504 322460 : info->inner_loop_cond = NULL;
1505 322460 : if (!loop->inner)
1506 : {
1507 : /* Inner-most loop. */
1508 :
1509 302375 : if (empty_block_p (loop->header))
1510 0 : return opt_result::failure_at (vect_location,
1511 : "not vectorized: empty loop.\n");
1512 : }
1513 : else
1514 : {
1515 20085 : class loop *innerloop = loop->inner;
1516 20085 : edge entryedge;
1517 :
1518 : /* Nested loop. We currently require that the loop is doubly-nested,
1519 : contains a single inner loop with a single exit to the block
1520 : with the single exit condition in the outer loop.
1521 : Vectorizable outer-loops look like this:
1522 :
1523 : (pre-header)
1524 : |
1525 : header <---+
1526 : | |
1527 : inner-loop |
1528 : | |
1529 : tail ------+
1530 : |
1531 : (exit-bb)
1532 :
1533 : The inner-loop also has the properties expected of inner-most loops
1534 : as described above. */
1535 :
1536 20085 : if ((loop->inner)->inner || (loop->inner)->next)
1537 3092 : return opt_result::failure_at (vect_location,
1538 : "not vectorized:"
1539 : " multiple nested loops.\n");
1540 :
1541 16993 : entryedge = loop_preheader_edge (innerloop);
1542 16993 : if (entryedge->src != loop->header
1543 16462 : || !single_exit (innerloop)
1544 28625 : || single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
1545 5672 : return opt_result::failure_at (vect_location,
1546 : "not vectorized:"
1547 : " unsupported outerloop form.\n");
1548 :
1549 : /* Analyze the inner-loop. */
1550 11321 : vect_loop_form_info inner;
1551 11321 : opt_result res = vect_analyze_loop_form (loop->inner, NULL, &inner);
1552 11321 : if (!res)
1553 : {
1554 417 : if (dump_enabled_p ())
1555 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1556 : "not vectorized: Bad inner loop.\n");
1557 417 : return res;
1558 : }
1559 :
1560 : /* Don't support analyzing niter under assumptions for inner
1561 : loop. */
1562 10904 : if (!integer_onep (inner.assumptions))
1563 263 : return opt_result::failure_at (vect_location,
1564 : "not vectorized: Bad inner loop.\n");
1565 :
1566 10641 : if (inner.number_of_iterations == chrec_dont_know
1567 10641 : || !expr_invariant_in_loop_p (loop, inner.number_of_iterations))
1568 1857 : return opt_result::failure_at (vect_location,
1569 : "not vectorized: inner-loop count not"
1570 : " invariant.\n");
1571 :
1572 8784 : if (dump_enabled_p ())
1573 1051 : dump_printf_loc (MSG_NOTE, vect_location,
1574 : "Considering outer-loop vectorization.\n");
1575 8784 : info->inner_loop_cond = inner.conds[0];
1576 11321 : }
1577 :
1578 311159 : if (EDGE_COUNT (loop->header->preds) != 2)
1579 0 : return opt_result::failure_at (vect_location,
1580 : "not vectorized:"
1581 : " too many incoming edges.\n");
1582 :
1583 : /* We assume that the latch is empty. */
1584 311159 : basic_block latch = loop->latch;
1585 311159 : do
1586 : {
1587 311159 : if (!empty_block_p (latch)
1588 311159 : || !gimple_seq_empty_p (phi_nodes (latch)))
1589 20313 : return opt_result::failure_at (vect_location,
1590 : "not vectorized: latch block not "
1591 : "empty.\n");
1592 290846 : latch = single_pred (latch);
1593 : }
1594 581692 : while (single_succ_p (latch));
1595 :
1596 : /* Make sure there is no abnormal exit. */
1597 290846 : auto_vec<edge> exits = get_loop_exit_edges (loop);
1598 1290936 : for (edge e : exits)
1599 : {
1600 418431 : if (e->flags & EDGE_ABNORMAL)
1601 33 : return opt_result::failure_at (vect_location,
1602 : "not vectorized:"
1603 : " abnormal loop exit edge.\n");
1604 : }
1605 :
1606 290813 : info->conds
1607 290813 : = vect_get_loop_niters (loop, exit_e, &info->assumptions,
1608 : &info->number_of_iterations,
1609 290813 : &info->number_of_iterationsm1);
1610 290813 : if (info->conds.is_empty ())
1611 34 : return opt_result::failure_at
1612 34 : (vect_location,
1613 : "not vectorized: complicated exit condition.\n");
1614 :
1615 : /* Determine what the primary and alternate exit conds are. */
1616 709143 : for (unsigned i = 0; i < info->conds.length (); i++)
1617 : {
1618 418364 : gcond *cond = info->conds[i];
1619 418364 : if (exit_e->src == gimple_bb (cond))
1620 290779 : std::swap (info->conds[0], info->conds[i]);
1621 : }
1622 :
1623 290779 : if (chrec_contains_undetermined (info->number_of_iterations))
1624 : {
1625 60273 : if (dump_enabled_p ())
1626 261 : dump_printf_loc (MSG_NOTE, vect_location,
1627 : "Loop being analyzed as uncounted.\n");
1628 60273 : if (loop->inner)
1629 563 : return opt_result::failure_at
1630 563 : (vect_location,
1631 : "not vectorized: outer loop vectorization of uncounted loops"
1632 : " is unsupported.\n");
1633 59710 : return opt_result::success ();
1634 : }
1635 :
1636 230506 : if (integer_zerop (info->assumptions))
1637 4 : return opt_result::failure_at
1638 4 : (info->conds[0],
1639 : "not vectorized: number of iterations cannot be computed.\n");
1640 :
1641 230502 : if (integer_zerop (info->number_of_iterations))
1642 18 : return opt_result::failure_at
1643 18 : (info->conds[0],
1644 : "not vectorized: number of iterations = 0.\n");
1645 :
1646 230484 : if (!(tree_fits_shwi_p (info->number_of_iterations)
1647 126728 : && tree_to_shwi (info->number_of_iterations) > 0))
1648 : {
1649 103756 : if (dump_enabled_p ())
1650 : {
1651 2785 : dump_printf_loc (MSG_NOTE, vect_location,
1652 : "Symbolic number of iterations is ");
1653 2785 : dump_generic_expr (MSG_NOTE, TDF_DETAILS, info->number_of_iterations);
1654 2785 : dump_printf (MSG_NOTE, "\n");
1655 : }
1656 : }
1657 :
1658 230484 : return opt_result::success ();
1659 290846 : }
1660 :
1661 : /* Create a loop_vec_info for LOOP with SHARED and the
1662 : vect_analyze_loop_form result. */
1663 :
1664 : loop_vec_info
1665 541421 : vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
1666 : const vect_loop_form_info *info,
1667 : loop_vec_info orig_loop_info)
1668 : {
1669 541421 : loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared, info->bbs);
1670 541421 : LOOP_VINFO_NITERSM1 (loop_vinfo) = info->number_of_iterationsm1;
1671 541421 : LOOP_VINFO_NITERS (loop_vinfo) = info->number_of_iterations;
1672 541421 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = info->number_of_iterations;
1673 541421 : LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = orig_loop_info;
1674 541421 : if (orig_loop_info && LOOP_VINFO_EPILOGUE_P (orig_loop_info))
1675 404 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo)
1676 404 : = LOOP_VINFO_MAIN_LOOP_INFO (orig_loop_info);
1677 : else
1678 541017 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) = orig_loop_info;
1679 : /* Also record the assumptions for versioning. */
1680 541421 : if (!integer_onep (info->assumptions) && !orig_loop_info)
1681 20348 : LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = info->assumptions;
1682 :
1683 2404574 : for (gcond *cond : info->conds)
1684 : {
1685 780311 : stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
1686 : /* Mark the statement as a condition. */
1687 780311 : STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
1688 : }
1689 :
1690 541421 : unsigned cond_id = 0;
1691 541421 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
1692 476576 : LOOP_VINFO_LOOP_IV_COND (loop_vinfo) = info->conds[cond_id++];
1693 :
1694 845156 : for (; cond_id < info->conds.length (); cond_id ++)
1695 303735 : LOOP_VINFO_LOOP_CONDS (loop_vinfo).safe_push (info->conds[cond_id]);
1696 :
1697 541421 : LOOP_VINFO_MAIN_EXIT (loop_vinfo) = info->loop_exit;
1698 :
1699 : /* Check to see if we're vectorizing multiple exits. */
1700 541421 : LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
1701 541421 : = !LOOP_VINFO_LOOP_CONDS (loop_vinfo).is_empty ();
1702 :
1703 : /* At the moment we can't support no epilogs for multiple exits, result of
1704 : the first compare should be masked by that of the second. We can only
1705 : allow it if the early exits have the same live values. for differing
1706 : values we have to calculate a third mask to disambiguate. */
1707 541421 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo)
1708 541421 : = LOOP_VINFO_LOOP_CONDS (loop_vinfo).length () > 1;
1709 :
1710 541421 : if (info->inner_loop_cond)
1711 : {
1712 : /* If we have an estimate on the number of iterations of the inner
1713 : loop use that to limit the scale for costing, otherwise use
1714 : --param vect-inner-loop-cost-factor literally. */
1715 7080 : widest_int nit;
1716 7080 : if (estimated_stmt_executions (loop->inner, &nit))
1717 5801 : LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo)
1718 5801 : = wi::smin (nit, param_vect_inner_loop_cost_factor).to_uhwi ();
1719 7080 : }
1720 :
1721 541421 : return loop_vinfo;
1722 : }
1723 :
1724 :
1725 :
1726 : /* Return true if we know that the iteration count is smaller than the
1727 : vectorization factor. Return false if it isn't, or if we can't be sure
1728 : either way. */
1729 :
1730 : static bool
1731 159576 : vect_known_niters_smaller_than_vf (loop_vec_info loop_vinfo)
1732 : {
1733 159576 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1734 :
1735 159576 : HOST_WIDE_INT max_niter;
1736 159576 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1737 82005 : max_niter = LOOP_VINFO_INT_NITERS (loop_vinfo);
1738 : else
1739 77571 : max_niter = max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo));
1740 :
1741 159576 : if (max_niter != -1 && (unsigned HOST_WIDE_INT) max_niter < assumed_vf)
1742 11296 : return true;
1743 :
1744 : return false;
1745 : }
1746 :
1747 : /* Analyze the cost of the loop described by LOOP_VINFO. Decide if it
1748 : is worthwhile to vectorize. Return 1 if definitely yes, 0 if
1749 : definitely no, or -1 if it's worth retrying. */
1750 :
1751 : static int
1752 159590 : vect_analyze_loop_costing (loop_vec_info loop_vinfo,
1753 : unsigned *suggested_unroll_factor)
1754 : {
1755 159590 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1756 159590 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1757 :
1758 : /* Only loops that can handle partially-populated vectors can have iteration
1759 : counts less than the vectorization factor. */
1760 159590 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
1761 159590 : && vect_known_niters_smaller_than_vf (loop_vinfo))
1762 : {
1763 11286 : if (dump_enabled_p ())
1764 258 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1765 : "not vectorized: iteration count smaller than "
1766 : "vectorization factor.\n");
1767 : return 0;
1768 : }
1769 :
1770 : /* If we know the number of iterations we can do better, for the
1771 : epilogue we can also decide whether the main loop leaves us
1772 : with enough iterations, preferring a smaller vector epilog then
1773 : also possibly used for the case we skip the vector loop. */
1774 148304 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1775 : {
1776 72019 : widest_int scalar_niters
1777 72019 : = wi::to_widest (LOOP_VINFO_NITERSM1 (loop_vinfo)) + 1;
1778 72019 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1779 : {
1780 2855 : loop_vec_info orig_loop_vinfo
1781 : = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
1782 2855 : loop_vec_info main_loop_vinfo
1783 : = LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo);
1784 2855 : unsigned lowest_vf
1785 2855 : = constant_lower_bound (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo));
1786 2855 : int prolog_peeling = 0;
1787 2855 : if (!vect_use_loop_mask_for_alignment_p (main_loop_vinfo))
1788 2855 : prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
1789 2855 : if (prolog_peeling >= 0
1790 2855 : && known_eq (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
1791 : lowest_vf))
1792 : {
1793 5698 : unsigned gap
1794 2849 : = LOOP_VINFO_PEELING_FOR_GAPS (main_loop_vinfo) ? 1 : 0;
1795 5698 : scalar_niters = ((scalar_niters - gap - prolog_peeling)
1796 5698 : % lowest_vf + gap);
1797 : }
1798 : }
1799 : /* Reject vectorizing for a single scalar iteration, even if
1800 : we could in principle implement that using partial vectors.
1801 : But allow such vectorization if VF == 1 in case we do not
1802 : need to peel for gaps (if we need, avoid vectorization for
1803 : reasons of code footprint). */
1804 72019 : unsigned peeling_gap = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo);
1805 72019 : if (scalar_niters <= peeling_gap + 1
1806 72019 : && (assumed_vf > 1 || peeling_gap != 0))
1807 : {
1808 686 : if (dump_enabled_p ())
1809 162 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1810 : "not vectorized: loop only has a single "
1811 : "scalar iteration.\n");
1812 : return 0;
1813 : }
1814 :
1815 71333 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
1816 : {
1817 : /* Check that the loop processes at least one full vector. */
1818 71320 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1819 71320 : if (known_lt (scalar_niters, vf))
1820 : {
1821 401 : if (dump_enabled_p ())
1822 347 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1823 : "loop does not have enough iterations "
1824 : "to support vectorization.\n");
1825 441 : return 0;
1826 : }
1827 :
1828 : /* If we need to peel an extra epilogue iteration to handle data
1829 : accesses with gaps, check that there are enough scalar iterations
1830 : available.
1831 :
1832 : The check above is redundant with this one when peeling for gaps,
1833 : but the distinction is useful for diagnostics. */
1834 70919 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
1835 71240 : && known_le (scalar_niters, vf))
1836 : {
1837 40 : if (dump_enabled_p ())
1838 9 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1839 : "loop does not have enough iterations "
1840 : "to support peeling for gaps.\n");
1841 : return 0;
1842 : }
1843 : }
1844 72019 : }
1845 :
1846 : /* If using the "very cheap" model. reject cases in which we'd keep
1847 : a copy of the scalar code (even if we might be able to vectorize it). */
1848 147177 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1849 147177 : && (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1850 77313 : || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
1851 : {
1852 721 : if (dump_enabled_p ())
1853 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1854 : "some scalar iterations would need to be peeled\n");
1855 : return 0;
1856 : }
1857 :
1858 146456 : int min_profitable_iters, min_profitable_estimate;
1859 146456 : vect_estimate_min_profitable_iters (loop_vinfo, &min_profitable_iters,
1860 : &min_profitable_estimate,
1861 : suggested_unroll_factor);
1862 :
1863 146456 : if (min_profitable_iters < 0)
1864 : {
1865 24962 : if (dump_enabled_p ())
1866 38 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1867 : "not vectorized: vectorization not profitable.\n");
1868 24962 : if (dump_enabled_p ())
1869 38 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1870 : "not vectorized: vector version will never be "
1871 : "profitable.\n");
1872 : return -1;
1873 : }
1874 :
1875 121494 : int min_scalar_loop_bound = (param_min_vect_loop_bound
1876 121494 : * assumed_vf);
1877 :
1878 : /* Use the cost model only if it is more conservative than user specified
1879 : threshold. */
1880 121494 : unsigned int th = (unsigned) MAX (min_scalar_loop_bound,
1881 : min_profitable_iters);
1882 :
1883 121494 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = th;
1884 :
1885 65426 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1886 186920 : && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
1887 : {
1888 597 : if (dump_enabled_p ())
1889 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1890 : "not vectorized: vectorization not profitable.\n");
1891 597 : if (dump_enabled_p ())
1892 2 : dump_printf_loc (MSG_NOTE, vect_location,
1893 : "not vectorized: iteration count smaller than user "
1894 : "specified loop bound parameter or minimum profitable "
1895 : "iterations (whichever is more conservative).\n");
1896 : return 0;
1897 : }
1898 :
1899 : /* The static profitablity threshold min_profitable_estimate includes
1900 : the cost of having to check at runtime whether the scalar loop
1901 : should be used instead. If it turns out that we don't need or want
1902 : such a check, the threshold we should use for the static estimate
1903 : is simply the point at which the vector loop becomes more profitable
1904 : than the scalar loop. */
1905 120897 : if (min_profitable_estimate > min_profitable_iters
1906 27792 : && !LOOP_REQUIRES_VERSIONING (loop_vinfo)
1907 27208 : && !LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
1908 644 : && !LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1909 121541 : && !vect_apply_runtime_profitability_check_p (loop_vinfo))
1910 : {
1911 14 : if (dump_enabled_p ())
1912 8 : dump_printf_loc (MSG_NOTE, vect_location, "no need for a runtime"
1913 : " choice between the scalar and vector loops\n");
1914 14 : min_profitable_estimate = min_profitable_iters;
1915 : }
1916 :
1917 : /* If the vector loop needs multiple iterations to be beneficial then
1918 : things are probably too close to call, and the conservative thing
1919 : would be to stick with the scalar code. */
1920 120897 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1921 61740 : && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1922 144700 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1923 : {
1924 15106 : if (dump_enabled_p ())
1925 225 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1926 : "one iteration of the vector loop would be"
1927 : " more expensive than the equivalent number of"
1928 : " iterations of the scalar loop\n");
1929 : return 0;
1930 : }
1931 :
1932 105791 : HOST_WIDE_INT estimated_niter;
1933 :
1934 : /* If we are vectorizing an epilogue then we know the maximum number of
1935 : scalar iterations it will cover is at least one lower than the
1936 : vectorization factor of the main loop. */
1937 105791 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1938 12554 : estimated_niter
1939 12554 : = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
1940 : else
1941 : {
1942 93237 : estimated_niter = estimated_stmt_executions_int (loop);
1943 93237 : if (estimated_niter == -1)
1944 30604 : estimated_niter = likely_max_stmt_executions_int (loop);
1945 : }
1946 43158 : if (estimated_niter != -1
1947 102852 : && ((unsigned HOST_WIDE_INT) estimated_niter
1948 102852 : < MAX (th, (unsigned) min_profitable_estimate)))
1949 : {
1950 4651 : if (dump_enabled_p ())
1951 34 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1952 : "not vectorized: estimated iteration count too "
1953 : "small.\n");
1954 4651 : if (dump_enabled_p ())
1955 34 : dump_printf_loc (MSG_NOTE, vect_location,
1956 : "not vectorized: estimated iteration count smaller "
1957 : "than specified loop bound parameter or minimum "
1958 : "profitable iterations (whichever is more "
1959 : "conservative).\n");
1960 : return -1;
1961 : }
1962 :
1963 : /* As we cannot use a runtime check to gate profitability for uncounted
1964 : loops require either an estimate or if none, at least a profitable
1965 : vectorization within the first vector iteration (that condition
1966 : will practically never be true due to the required epilog and
1967 : likely alignment prologue). */
1968 101140 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
1969 165 : && estimated_niter == -1
1970 101278 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1971 : {
1972 120 : if (dump_enabled_p ())
1973 2 : dump_printf_loc (MSG_NOTE, vect_location,
1974 : "not vectorized: no loop iteration estimate on the "
1975 : "uncounted loop and not trivially profitable.\n");
1976 : return -1;
1977 : }
1978 :
1979 : return 1;
1980 : }
1981 :
1982 : /* Gather data references in LOOP with body BBS and store them into
1983 : *DATAREFS. */
1984 :
1985 : static opt_result
1986 278593 : vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
1987 : vec<data_reference_p> *datarefs)
1988 : {
1989 833233 : for (unsigned i = 0; i < loop->num_nodes; i++)
1990 1235674 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
1991 5417932 : !gsi_end_p (gsi); gsi_next (&gsi))
1992 : {
1993 4863292 : gimple *stmt = gsi_stmt (gsi);
1994 4863292 : if (is_gimple_debug (stmt))
1995 2321077 : continue;
1996 2542351 : opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs,
1997 : NULL, 0);
1998 2542351 : if (!res)
1999 : {
2000 63333 : if (is_gimple_call (stmt) && loop->safelen)
2001 : {
2002 412 : tree fndecl = gimple_call_fndecl (stmt), op;
2003 412 : if (fndecl == NULL_TREE
2004 412 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
2005 : {
2006 0 : fndecl = gimple_call_arg (stmt, 0);
2007 0 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
2008 0 : fndecl = TREE_OPERAND (fndecl, 0);
2009 0 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
2010 : }
2011 412 : if (fndecl != NULL_TREE)
2012 : {
2013 375 : cgraph_node *node = cgraph_node::get (fndecl);
2014 375 : if (node != NULL && node->simd_clones != NULL)
2015 : {
2016 137 : unsigned int j, n = gimple_call_num_args (stmt);
2017 565 : for (j = 0; j < n; j++)
2018 : {
2019 292 : op = gimple_call_arg (stmt, j);
2020 292 : if (DECL_P (op)
2021 292 : || (REFERENCE_CLASS_P (op)
2022 0 : && get_base_address (op)))
2023 : break;
2024 : }
2025 137 : op = gimple_call_lhs (stmt);
2026 : /* Ignore #pragma omp declare simd functions
2027 : if they don't have data references in the
2028 : call stmt itself. */
2029 273 : if (j == n
2030 137 : && !(op
2031 124 : && (DECL_P (op)
2032 124 : || (REFERENCE_CLASS_P (op)
2033 0 : && get_base_address (op)))))
2034 136 : continue;
2035 : }
2036 : }
2037 : }
2038 63197 : return res;
2039 : }
2040 : /* If dependence analysis will give up due to the limit on the
2041 : number of datarefs stop here and fail fatally. */
2042 4385150 : if (datarefs->length ()
2043 1906132 : > (unsigned)param_loop_max_datarefs_for_datadeps)
2044 0 : return opt_result::failure_at (stmt, "exceeded param "
2045 : "loop-max-datarefs-for-datadeps\n");
2046 : }
2047 215396 : return opt_result::success ();
2048 : }
2049 :
2050 : /* Gather no-wrap bounds from SHARED's datarefs and accumulate them in
2051 : one assumption. Validate this assumption, as well as INFO's niter
2052 : assumption and return an error on failure.
2053 : If the assumptions are valid, reset loop properties as well as
2054 : scev's hashtable and return success. */
2055 :
2056 : static opt_result
2057 215396 : vect_build_assumptions (class loop *loop, vec_info_shared *shared,
2058 : vect_loop_form_info *info)
2059 : {
2060 215396 : tree niter = info->number_of_iterations;
2061 215396 : bool niter_known = niter && !chrec_contains_undetermined (niter);
2062 :
2063 215396 : tree nowrap_assum = NULL_TREE;
2064 215396 : tree bound_min = NULL_TREE;
2065 :
2066 215396 : unsigned int i;
2067 215396 : struct data_reference *dr;
2068 :
2069 : /* Build the minimum over all nowrap bounds. */
2070 763925 : FOR_EACH_VEC_ELT (shared->datarefs, i, dr)
2071 : {
2072 548529 : tree bound = DR_NOWRAP_BOUND (dr);
2073 548529 : if (!bound)
2074 542285 : continue;
2075 :
2076 : /* Use scev's function to MIN two bounds. */
2077 6244 : scev_combine_nowrap_bounds (&bound_min, bound);
2078 : }
2079 :
2080 : /* Build "NITER < BOUND_MIN". */
2081 215396 : if (niter_known && bound_min)
2082 : {
2083 979 : unsigned prec = MAX (TYPE_PRECISION (TREE_TYPE (niter)),
2084 : TYPE_PRECISION (TREE_TYPE (bound_min)));
2085 979 : tree t = build_nonstandard_integer_type (prec, 1);
2086 979 : nowrap_assum = fold_build2 (LE_EXPR, boolean_type_node,
2087 : fold_convert (t, niter),
2088 : fold_convert (t, bound_min));
2089 :
2090 : /* Try to simplify the assumption with ranger. */
2091 979 : value_range vr (TREE_TYPE (nowrap_assum));
2092 979 : tree better_assum;
2093 979 : if (get_range_query (cfun)->range_on_edge
2094 979 : (vr, loop_preheader_edge (loop), nowrap_assum)
2095 1958 : && vr.singleton_p (&better_assum))
2096 188 : nowrap_assum = better_assum;
2097 979 : }
2098 :
2099 : /* Combine nowrap assumptions and niter assumption. */
2100 979 : if (nowrap_assum)
2101 : {
2102 979 : if (dump_enabled_p ())
2103 16 : dump_printf_loc (MSG_NOTE, vect_location,
2104 : "adding no-wrap assumption: %T\n", nowrap_assum);
2105 :
2106 979 : if (info->assumptions && !integer_onep (info->assumptions))
2107 253 : nowrap_assum = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
2108 : info->assumptions, nowrap_assum);
2109 979 : info->assumptions = nowrap_assum;
2110 : }
2111 :
2112 215396 : if (integer_zerop (info->assumptions))
2113 129 : return opt_result::failure_at
2114 129 : (vect_location,
2115 : "not vectorized: number of iterations cannot be computed.\n");
2116 :
2117 215267 : if (!integer_onep (info->assumptions))
2118 : {
2119 7852 : if (dump_enabled_p ())
2120 : {
2121 84 : dump_printf_loc (MSG_NOTE, vect_location,
2122 : "Loop to be versioned with assumption ");
2123 84 : dump_generic_expr (MSG_NOTE, TDF_SLIM, info->assumptions);
2124 84 : dump_printf (MSG_NOTE, "\n");
2125 : }
2126 :
2127 : /* We consider to vectorize this loop by versioning it under
2128 : some assumptions. In order to do this, we need to clear
2129 : existing information computed by scev and niter analyzer. */
2130 7852 : scev_reset_htab ();
2131 7852 : free_numbers_of_iterations_estimates (loop);
2132 : /* Also set flag for this loop so that following scev and niter
2133 : analysis are done under the assumptions. */
2134 7852 : loop_constraint_set (loop, LOOP_C_FINITE);
2135 : }
2136 : else
2137 : /* Clear the existing niter information to make sure the nonwrapping flag
2138 : will be calculated and set propriately. */
2139 207415 : free_numbers_of_iterations_estimates (loop);
2140 :
2141 215267 : return opt_result::success ();
2142 : }
2143 :
2144 :
2145 : /* Determine if operating on full vectors for LOOP_VINFO might leave
2146 : some scalar iterations still to do. If so, decide how we should
2147 : handle those scalar iterations. The possibilities are:
2148 :
2149 : (1) Make LOOP_VINFO operate on partial vectors instead of full vectors.
2150 : In this case:
2151 :
2152 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == true
2153 : LOOP_VINFO_PEELING_FOR_NITER == false
2154 :
2155 : (2) Make LOOP_VINFO operate on full vectors and use an epilogue loop
2156 : to handle the remaining scalar iterations. In this case:
2157 :
2158 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == false
2159 : LOOP_VINFO_PEELING_FOR_NITER == true
2160 :
2161 : The MASKED_P argument specifies to what extent
2162 : param_vect_partial_vector_usage is to be honored. For MASKED_P == 0
2163 : no partial vectors are to be used, for MASKED_P == -1 it's
2164 : param_vect_partial_vector_usage that gets to decide whether we may
2165 : consider partial vector usage. For MASKED_P == 1 partial vectors
2166 : may be used if possible.
2167 :
2168 : */
2169 :
2170 : static opt_result
2171 160574 : vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
2172 : int masked_p)
2173 : {
2174 : /* Determine whether there would be any scalar iterations left over. */
2175 160574 : bool need_peeling_or_partial_vectors_p
2176 160574 : = vect_need_peeling_or_partial_vectors_p (loop_vinfo);
2177 :
2178 : /* Decide whether to vectorize the loop with partial vectors. */
2179 160574 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2180 160574 : if (masked_p == 0
2181 160574 : || (masked_p == -1 && param_vect_partial_vector_usage == 0))
2182 : /* If requested explicitly do not use partial vectors. */
2183 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2184 227 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2185 71 : && LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo))
2186 0 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2187 227 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2188 71 : && need_peeling_or_partial_vectors_p)
2189 : {
2190 : /* For partial-vector-usage=1, try to push the handling of partial
2191 : vectors to the epilogue, with the main loop continuing to operate
2192 : on full vectors.
2193 :
2194 : If we are unrolling we also do not want to use partial vectors. This
2195 : is to avoid the overhead of generating multiple masks and also to
2196 : avoid having to execute entire iterations of FALSE masked instructions
2197 : when dealing with one or less full iterations.
2198 :
2199 : ??? We could then end up failing to use partial vectors if we
2200 : decide to peel iterations into a prologue, and if the main loop
2201 : then ends up processing fewer than VF iterations. */
2202 48 : if ((param_vect_partial_vector_usage == 1
2203 15 : || loop_vinfo->suggested_unroll_factor > 1)
2204 33 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2205 70 : && !vect_known_niters_smaller_than_vf (loop_vinfo))
2206 : ;
2207 : else
2208 36 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2209 : }
2210 :
2211 160574 : if (LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo)
2212 0 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2213 0 : return opt_result::failure_at (vect_location,
2214 : "not vectorized: loop needs but cannot "
2215 : "use partial vectors\n");
2216 :
2217 160574 : if (dump_enabled_p ())
2218 13204 : dump_printf_loc (MSG_NOTE, vect_location,
2219 : "operating on %s vectors%s.\n",
2220 13204 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2221 : ? "partial" : "full",
2222 13204 : LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2223 : ? " for epilogue loop" : "");
2224 :
2225 160574 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2226 321148 : = (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2227 160574 : && need_peeling_or_partial_vectors_p);
2228 :
2229 160574 : return opt_result::success ();
2230 : }
2231 :
2232 : /* Function vect_analyze_loop_2.
2233 :
2234 : Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
2235 : analyses will record information in some members of LOOP_VINFO. FATAL
2236 : indicates if some analysis meets fatal error. If one non-NULL pointer
2237 : SUGGESTED_UNROLL_FACTOR is provided, it's intent to be filled with one
2238 : worked out suggested unroll factor, while one NULL pointer shows it's
2239 : going to apply the suggested unroll factor.
2240 : SINGLE_LANE_SLP_DONE_FOR_SUGGESTED_UF is to hold whether single-lane
2241 : slp was forced when the suggested unroll factor was worked out. */
2242 : static opt_result
2243 540724 : vect_analyze_loop_2 (loop_vec_info loop_vinfo, int masked_p, bool &fatal,
2244 : unsigned *suggested_unroll_factor,
2245 : bool& single_lane_slp_done_for_suggested_uf)
2246 : {
2247 540724 : opt_result ok = opt_result::success ();
2248 540724 : int res;
2249 540724 : unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
2250 540724 : loop_vec_info orig_loop_vinfo = NULL;
2251 :
2252 : /* If we are dealing with an epilogue then orig_loop_vinfo points to the
2253 : loop_vec_info of the first vectorized loop. */
2254 540724 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2255 14495 : orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
2256 : else
2257 : orig_loop_vinfo = loop_vinfo;
2258 14495 : gcc_assert (orig_loop_vinfo);
2259 :
2260 : /* We can't mask on niters for uncounted loops due to unknown upper bound. */
2261 540724 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
2262 64845 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2263 :
2264 : /* The first group of checks is independent of the vector size. */
2265 540724 : fatal = true;
2266 :
2267 540724 : if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)
2268 540724 : && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)))
2269 5 : return opt_result::failure_at (vect_location,
2270 : "not vectorized: simd if(0)\n");
2271 :
2272 : /* Find all data references in the loop (which correspond to vdefs/vuses)
2273 : and analyze their evolution in the loop. */
2274 :
2275 540719 : loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
2276 :
2277 : /* Check the data references. */
2278 540719 : loop_vinfo->shared->check_datarefs ();
2279 :
2280 : /* Analyze the data references and also adjust the minimal
2281 : vectorization factor according to the loads and stores. */
2282 :
2283 540719 : ok = vect_analyze_data_refs (loop_vinfo, &fatal);
2284 540719 : if (!ok)
2285 : {
2286 75849 : if (dump_enabled_p ())
2287 1375 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2288 : "bad data references.\n");
2289 75849 : return ok;
2290 : }
2291 :
2292 : /* Check if we are applying unroll factor now. */
2293 464870 : bool applying_suggested_uf = loop_vinfo->suggested_unroll_factor > 1;
2294 464870 : gcc_assert (!applying_suggested_uf || !suggested_unroll_factor);
2295 :
2296 : /* When single-lane SLP was forced and we are applying suggested unroll
2297 : factor, keep that decision here. */
2298 929740 : bool force_single_lane = (applying_suggested_uf
2299 464870 : && single_lane_slp_done_for_suggested_uf);
2300 :
2301 : /* Classify all cross-iteration scalar data-flow cycles.
2302 : Cross-iteration cycles caused by virtual phis are analyzed separately. */
2303 464870 : vect_analyze_scalar_cycles (loop_vinfo);
2304 :
2305 464870 : vect_pattern_recog (loop_vinfo);
2306 :
2307 : /* Analyze the access patterns of the data-refs in the loop (consecutive,
2308 : complex, etc.). FORNOW: Only handle consecutive access pattern. */
2309 :
2310 464870 : ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
2311 464870 : if (!ok)
2312 : {
2313 8172 : if (dump_enabled_p ())
2314 292 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2315 : "bad data access.\n");
2316 8172 : return ok;
2317 : }
2318 :
2319 : /* Data-flow analysis to detect stmts that do not need to be vectorized. */
2320 :
2321 456698 : ok = vect_mark_stmts_to_be_vectorized (loop_vinfo, &fatal);
2322 456698 : if (!ok)
2323 : {
2324 47564 : if (dump_enabled_p ())
2325 409 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2326 : "unexpected pattern.\n");
2327 47564 : return ok;
2328 : }
2329 :
2330 : /* While the rest of the analysis below depends on it in some way. */
2331 409134 : fatal = false;
2332 :
2333 : /* Analyze data dependences between the data-refs in the loop
2334 : and adjust the maximum vectorization factor according to
2335 : the dependences.
2336 : FORNOW: fail at the first data dependence that we encounter. */
2337 :
2338 409134 : ok = vect_analyze_data_ref_dependences (loop_vinfo, &max_vf);
2339 409134 : if (!ok)
2340 : {
2341 22492 : if (dump_enabled_p ())
2342 563 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2343 : "bad data dependence.\n");
2344 22492 : return ok;
2345 : }
2346 386642 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
2347 :
2348 : /* Compute the scalar iteration cost. */
2349 386642 : vect_compute_single_scalar_iteration_cost (loop_vinfo);
2350 :
2351 386642 : bool saved_can_use_partial_vectors_p
2352 : = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
2353 :
2354 : /* This is the point where we can re-start analysis with single-lane
2355 : SLP forced. */
2356 537778 : start_over:
2357 :
2358 : /* Check the SLP opportunities in the loop, analyze and build
2359 : SLP trees. */
2360 1075556 : ok = vect_analyze_slp (loop_vinfo, loop_vinfo->stmt_vec_infos.length (),
2361 : force_single_lane);
2362 537778 : if (!ok)
2363 19785 : return ok;
2364 :
2365 : /* If there are any SLP instances mark them as pure_slp and compute
2366 : the overall vectorization factor. */
2367 517993 : if (!vect_make_slp_decision (loop_vinfo))
2368 68340 : return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
2369 :
2370 449653 : if (dump_enabled_p ())
2371 20930 : dump_printf_loc (MSG_NOTE, vect_location, "Loop contains only SLP stmts\n");
2372 :
2373 : /* Dump the vectorization factor from the SLP decision. */
2374 449653 : if (dump_enabled_p ())
2375 : {
2376 20930 : dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
2377 20930 : dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2378 20930 : dump_printf (MSG_NOTE, "\n");
2379 : }
2380 :
2381 : /* We don't expect to have to roll back to anything other than an empty
2382 : set of rgroups. */
2383 449653 : gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
2384 :
2385 : /* Apply the suggested unrolling factor, this was determined by the backend
2386 : during finish_cost the first time we ran the analysis for this
2387 : vector mode. */
2388 449653 : if (applying_suggested_uf)
2389 459 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) *= loop_vinfo->suggested_unroll_factor;
2390 :
2391 : /* Now the vectorization factor is final. */
2392 449653 : poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2393 449653 : gcc_assert (known_ne (vectorization_factor, 0U));
2394 :
2395 : /* Optimize the SLP graph with the vectorization factor fixed. */
2396 449653 : vect_optimize_slp (loop_vinfo);
2397 :
2398 : /* Gather the loads reachable from the SLP graph entries. */
2399 449653 : vect_gather_slp_loads (loop_vinfo);
2400 :
2401 449653 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
2402 : {
2403 15241 : dump_printf_loc (MSG_NOTE, vect_location,
2404 : "vectorization_factor = ");
2405 15241 : dump_dec (MSG_NOTE, vectorization_factor);
2406 15241 : dump_printf (MSG_NOTE, ", niters = %wd\n",
2407 15241 : LOOP_VINFO_INT_NITERS (loop_vinfo));
2408 : }
2409 :
2410 449653 : if (max_vf != MAX_VECTORIZATION_FACTOR
2411 449653 : && maybe_lt (max_vf, LOOP_VINFO_VECT_FACTOR (loop_vinfo)))
2412 41 : return opt_result::failure_at (vect_location, "bad data dependence.\n");
2413 :
2414 449612 : loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
2415 :
2416 : /* Analyze the alignment of the data-refs in the loop. */
2417 449612 : vect_analyze_data_refs_alignment (loop_vinfo);
2418 :
2419 : /* Prune the list of ddrs to be tested at run-time by versioning for alias.
2420 : It is important to call pruning after vect_analyze_data_ref_accesses,
2421 : since we use grouping information gathered by interleaving analysis. */
2422 449612 : ok = vect_prune_runtime_alias_test_list (loop_vinfo);
2423 449612 : if (!ok)
2424 17786 : return ok;
2425 :
2426 : /* Do not invoke vect_enhance_data_refs_alignment for epilogue
2427 : vectorization, since we do not want to add extra peeling or
2428 : add versioning for alignment. */
2429 431826 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2430 : /* This pass will decide on using loop versioning and/or loop peeling in
2431 : order to enhance the alignment of data references in the loop. */
2432 416353 : ok = vect_enhance_data_refs_alignment (loop_vinfo);
2433 431826 : if (!ok)
2434 0 : return ok;
2435 :
2436 : /* Analyze operations in the SLP instances. We can't simply
2437 : remove unsupported SLP instances as this makes the above
2438 : SLP kind detection invalid and might also affect the VF. */
2439 431826 : if (! vect_slp_analyze_operations (loop_vinfo))
2440 : {
2441 271252 : ok = opt_result::failure_at (vect_location,
2442 : "unsupported SLP instances\n");
2443 271252 : goto again;
2444 : }
2445 :
2446 : /* For now, we don't expect to mix both masking and length approaches for one
2447 : loop, disable it if both are recorded. */
2448 160574 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2449 24138 : && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ()
2450 184706 : && !LOOP_VINFO_LENS (loop_vinfo).is_empty ())
2451 : {
2452 0 : if (dump_enabled_p ())
2453 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2454 : "can't vectorize a loop with partial vectors"
2455 : " because we don't expect to mix different"
2456 : " approaches with partial vectors for the"
2457 : " same loop.\n");
2458 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2459 : }
2460 :
2461 : /* If we still have the option of using partial vectors,
2462 : check whether we can generate the necessary loop controls. */
2463 160574 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
2464 : {
2465 24138 : if (!LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
2466 : {
2467 24132 : if (!vect_verify_full_masking (loop_vinfo)
2468 24132 : && !vect_verify_full_masking_avx512 (loop_vinfo))
2469 6158 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2470 : }
2471 : else /* !LOOP_VINFO_LENS (loop_vinfo).is_empty () */
2472 6 : if (!vect_verify_loop_lens (loop_vinfo))
2473 6 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2474 : }
2475 :
2476 : /* Decide whether this loop_vinfo should use partial vectors or peeling,
2477 : assuming that the loop will be used as a main loop. We will redo
2478 : this analysis later if we instead decide to use the loop as an
2479 : epilogue loop. */
2480 160574 : ok = vect_determine_partial_vectors_and_peeling (loop_vinfo, masked_p);
2481 160574 : if (!ok)
2482 0 : return ok;
2483 :
2484 : /* If we're vectorizing a loop that uses length "controls" and
2485 : can iterate more than once, we apply decrementing IV approach
2486 : in loop control. */
2487 160574 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2488 36 : && LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_len
2489 0 : && LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) == 0
2490 160574 : && !(LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2491 0 : && known_le (LOOP_VINFO_INT_NITERS (loop_vinfo),
2492 : LOOP_VINFO_VECT_FACTOR (loop_vinfo))))
2493 0 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = true;
2494 :
2495 : /* If a loop uses length controls and has a decrementing loop control IV,
2496 : we will normally pass that IV through a MIN_EXPR to calcaluate the
2497 : basis for the length controls. E.g. in a loop that processes one
2498 : element per scalar iteration, the number of elements would be
2499 : MIN_EXPR <N, VF>, where N is the number of scalar iterations left.
2500 :
2501 : This MIN_EXPR approach allows us to use pointer IVs with an invariant
2502 : step, since only the final iteration of the vector loop can have
2503 : inactive lanes.
2504 :
2505 : However, some targets have a dedicated instruction for calculating the
2506 : preferred length, given the total number of elements that still need to
2507 : be processed. This is encapsulated in the SELECT_VL internal function.
2508 :
2509 : If the target supports SELECT_VL, we can use it instead of MIN_EXPR
2510 : to determine the basis for the length controls. However, unlike the
2511 : MIN_EXPR calculation, the SELECT_VL calculation can decide to make
2512 : lanes inactive in any iteration of the vector loop, not just the last
2513 : iteration. This SELECT_VL approach therefore requires us to use pointer
2514 : IVs with variable steps.
2515 :
2516 : Once we've decided how many elements should be processed by one
2517 : iteration of the vector loop, we need to populate the rgroup controls.
2518 : If a loop has multiple rgroups, we need to make sure that those rgroups
2519 : "line up" (that is, they must be consistent about which elements are
2520 : active and which aren't). This is done by vect_adjust_loop_lens_control.
2521 :
2522 : In principle, it would be possible to use vect_adjust_loop_lens_control
2523 : on either the result of a MIN_EXPR or the result of a SELECT_VL.
2524 : However:
2525 :
2526 : (1) In practice, it only makes sense to use SELECT_VL when a vector
2527 : operation will be controlled directly by the result. It is not
2528 : worth using SELECT_VL if it would only be the input to other
2529 : calculations.
2530 :
2531 : (2) If we use SELECT_VL for an rgroup that has N controls, each associated
2532 : pointer IV will need N updates by a variable amount (N-1 updates
2533 : within the iteration and 1 update to move to the next iteration).
2534 :
2535 : Because of this, we prefer to use the MIN_EXPR approach whenever there
2536 : is more than one length control.
2537 :
2538 : In addition, SELECT_VL always operates to a granularity of 1 unit.
2539 : If we wanted to use it to control an SLP operation on N consecutive
2540 : elements, we would need to make the SELECT_VL inputs measure scalar
2541 : iterations (rather than elements) and then multiply the SELECT_VL
2542 : result by N. But using SELECT_VL this way is inefficient because
2543 : of (1) above.
2544 :
2545 : 2. We don't apply SELECT_VL on single-rgroup when both (1) and (2) are
2546 : satisfied:
2547 :
2548 : (1). LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) is true.
2549 : (2). LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant () is true.
2550 :
2551 : Since SELECT_VL (variable step) will make SCEV analysis failed and then
2552 : we will fail to gain benefits of following unroll optimizations. We prefer
2553 : using the MIN_EXPR approach in this situation. */
2554 160574 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
2555 : {
2556 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
2557 0 : if (LOOP_VINFO_LENS (loop_vinfo).length () == 1
2558 0 : && LOOP_VINFO_LENS (loop_vinfo)[0].factor == 1
2559 0 : && (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2560 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ()))
2561 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = true;
2562 :
2563 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2564 0 : for (auto rgc : LOOP_VINFO_LENS (loop_vinfo))
2565 0 : if (rgc.type
2566 0 : && !direct_internal_fn_supported_p (IFN_SELECT_VL,
2567 : rgc.type, iv_type,
2568 : OPTIMIZE_FOR_SPEED))
2569 : {
2570 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2571 0 : break;
2572 : }
2573 :
2574 : /* If any of the SLP instances cover more than a single lane
2575 : we cannot use .SELECT_VL at the moment, even if the number
2576 : of lanes is uniform throughout the SLP graph. */
2577 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2578 0 : for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
2579 0 : if (SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)) != 1
2580 0 : && !(SLP_INSTANCE_KIND (inst) == slp_inst_kind_store
2581 0 : && SLP_INSTANCE_TREE (inst)->ldst_lanes))
2582 : {
2583 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2584 0 : break;
2585 : }
2586 : }
2587 :
2588 : /* If we're vectorizing an epilogue loop, the vectorized loop either needs
2589 : to be able to handle fewer than VF scalars, or needs to have a lower VF
2590 : than the main loop. */
2591 160574 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2592 14049 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2593 : {
2594 14035 : poly_uint64 unscaled_vf
2595 14035 : = exact_div (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
2596 : orig_loop_vinfo->suggested_unroll_factor);
2597 14035 : if (maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo), unscaled_vf))
2598 396 : return opt_result::failure_at (vect_location,
2599 : "Vectorization factor too high for"
2600 : " epilogue loop.\n");
2601 : }
2602 :
2603 : /* If the epilogue needs peeling for gaps but the main loop doesn't give
2604 : up on the epilogue. */
2605 160178 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2606 13653 : && LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2607 73 : && (LOOP_VINFO_PEELING_FOR_GAPS (orig_loop_vinfo)
2608 : != LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
2609 4 : return opt_result::failure_at (vect_location,
2610 : "Epilogue loop requires peeling for gaps "
2611 : "but main loop does not.\n");
2612 :
2613 : /* If an epilogue loop is required make sure we can create one. */
2614 160174 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2615 158868 : || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2616 58436 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2617 : {
2618 103325 : if (dump_enabled_p ())
2619 6072 : dump_printf_loc (MSG_NOTE, vect_location, "epilog loop required\n");
2620 103325 : if (!vect_can_advance_ivs_p (loop_vinfo)
2621 206066 : || !slpeel_can_duplicate_loop_p (loop,
2622 : LOOP_VINFO_MAIN_EXIT (loop_vinfo),
2623 102741 : LOOP_VINFO_MAIN_EXIT (loop_vinfo)))
2624 : {
2625 584 : ok = opt_result::failure_at (vect_location,
2626 : "not vectorized: can't create required "
2627 : "epilog loop\n");
2628 584 : goto again;
2629 : }
2630 : }
2631 :
2632 : /* Check the costings of the loop make vectorizing worthwhile. */
2633 159590 : res = vect_analyze_loop_costing (loop_vinfo, suggested_unroll_factor);
2634 159590 : if (res < 0 && !param_vect_allow_possibly_not_worthwhile_vectorizations)
2635 : {
2636 29733 : ok = opt_result::failure_at (vect_location,
2637 : "Loop costings may not be worthwhile.\n");
2638 29733 : goto again;
2639 : }
2640 129857 : if (!res)
2641 28837 : return opt_result::failure_at (vect_location,
2642 : "Loop costings not worthwhile.\n");
2643 :
2644 : /* During peeling, we need to check if number of loop iterations is
2645 : enough for both peeled prolog loop and vector loop. This check
2646 : can be merged along with threshold check of loop versioning, so
2647 : increase threshold for this case if necessary.
2648 :
2649 : If we are analyzing an epilogue we still want to check what its
2650 : versioning threshold would be. If we decide to vectorize the epilogues we
2651 : will want to use the lowest versioning threshold of all epilogues and main
2652 : loop. This will enable us to enter a vectorized epilogue even when
2653 : versioning the loop. We can't simply check whether the epilogue requires
2654 : versioning though since we may have skipped some versioning checks when
2655 : analyzing the epilogue. For instance, checks for alias versioning will be
2656 : skipped when dealing with epilogues as we assume we already checked them
2657 : for the main loop. So instead we always check the 'orig_loop_vinfo'. */
2658 101020 : if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo))
2659 : {
2660 9332 : poly_uint64 niters_th = 0;
2661 9332 : unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2662 :
2663 9332 : if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
2664 : {
2665 : /* Niters for peeled prolog loop. */
2666 9332 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
2667 : {
2668 115 : dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
2669 115 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
2670 115 : niters_th += TYPE_VECTOR_SUBPARTS (vectype) - 1;
2671 : }
2672 : else
2673 9217 : niters_th += LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
2674 : }
2675 :
2676 : /* Niters for one vector iteration, excluding a header-only iteration. */
2677 9332 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2678 9328 : niters_th += LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2679 9332 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
2680 48 : niters_th += 1;
2681 : /* One additional iteration because of peeling for gap. */
2682 9332 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
2683 75 : niters_th += 1;
2684 :
2685 : /* Use the same condition as vect_transform_loop to decide when to use
2686 : the cost to determine a versioning threshold. */
2687 9332 : if (vect_apply_runtime_profitability_check_p (loop_vinfo)
2688 9332 : && ordered_p (th, niters_th))
2689 6952 : niters_th = ordered_max (poly_uint64 (th), niters_th);
2690 :
2691 9332 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = niters_th;
2692 : }
2693 :
2694 101020 : gcc_assert (known_eq (vectorization_factor,
2695 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
2696 :
2697 101020 : single_lane_slp_done_for_suggested_uf = force_single_lane;
2698 :
2699 : /* Ok to vectorize! */
2700 101020 : LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
2701 101020 : return opt_result::success ();
2702 :
2703 301569 : again:
2704 : /* Ensure that "ok" is false (with an opt_problem if dumping is enabled). */
2705 301569 : gcc_assert (!ok);
2706 :
2707 : /* Try again with single-lane SLP. */
2708 301569 : if (force_single_lane)
2709 149451 : return ok;
2710 :
2711 : /* If we are applying suggested unroll factor, we don't need to
2712 : re-try any more as we want to keep the SLP mode fixed. */
2713 152118 : if (applying_suggested_uf)
2714 10 : return ok;
2715 :
2716 : /* Likewise if the grouped loads or stores in the SLP cannot be handled
2717 : via interleaving or lane instructions. */
2718 : slp_instance instance;
2719 : slp_tree node;
2720 : unsigned i, j;
2721 409017 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
2722 : {
2723 257881 : if (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance)) != vect_internal_def)
2724 0 : continue;
2725 :
2726 257881 : stmt_vec_info vinfo;
2727 257881 : vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0];
2728 257881 : if (!vinfo || !STMT_VINFO_GROUPED_ACCESS (vinfo))
2729 253286 : continue;
2730 4595 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2731 4595 : unsigned int size = DR_GROUP_SIZE (vinfo);
2732 4595 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
2733 4595 : if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
2734 8512 : && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
2735 9184 : && ! vect_grouped_store_supported (vectype, size))
2736 672 : return opt_result::failure_at (vinfo->stmt,
2737 : "unsupported grouped store\n");
2738 263263 : FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node)
2739 : {
2740 4156 : vinfo = SLP_TREE_REPRESENTATIVE (node);
2741 4156 : if (STMT_VINFO_GROUPED_ACCESS (vinfo))
2742 : {
2743 2026 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2744 2026 : bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
2745 2026 : size = DR_GROUP_SIZE (vinfo);
2746 2026 : vectype = SLP_TREE_VECTYPE (node);
2747 2026 : if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
2748 2026 : && ! vect_grouped_load_supported (vectype, single_element_p,
2749 : size))
2750 300 : return opt_result::failure_at (vinfo->stmt,
2751 : "unsupported grouped load\n");
2752 : }
2753 : }
2754 : }
2755 :
2756 : /* Roll back state appropriately. Force single-lane SLP this time. */
2757 151136 : force_single_lane = true;
2758 151136 : if (dump_enabled_p ())
2759 3994 : dump_printf_loc (MSG_NOTE, vect_location,
2760 : "re-trying with single-lane SLP\n");
2761 :
2762 : /* Reset the vectorization factor. */
2763 151136 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
2764 : /* Free the SLP instances. */
2765 408038 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
2766 256902 : vect_free_slp_instance (instance);
2767 151136 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
2768 : /* Reset altered state on stmts. */
2769 723120 : for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
2770 : {
2771 420848 : basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
2772 420848 : for (gimple_stmt_iterator si = gsi_start_phis (bb);
2773 760512 : !gsi_end_p (si); gsi_next (&si))
2774 : {
2775 339664 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
2776 339664 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
2777 339664 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
2778 : {
2779 : /* vectorizable_reduction adjusts reduction stmt def-types,
2780 : restore them to that of the PHI. */
2781 27004 : STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info))
2782 27004 : = STMT_VINFO_DEF_TYPE (stmt_info);
2783 27004 : STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
2784 : (STMT_VINFO_REDUC_DEF (stmt_info)))
2785 27004 : = STMT_VINFO_DEF_TYPE (stmt_info);
2786 : }
2787 : }
2788 : }
2789 : /* Free optimized alias test DDRS. */
2790 151136 : LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
2791 151136 : LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
2792 151136 : LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).release ();
2793 : /* Reset target cost data. */
2794 151136 : delete loop_vinfo->vector_costs;
2795 151136 : loop_vinfo->vector_costs = nullptr;
2796 : /* Reset accumulated rgroup information. */
2797 151136 : LOOP_VINFO_MASKS (loop_vinfo).mask_set.empty ();
2798 151136 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
2799 151136 : release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo));
2800 : /* Reset assorted flags. */
2801 151136 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo) = false;
2802 151136 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = false;
2803 151136 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = 0;
2804 151136 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = 0;
2805 151136 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2806 151136 : = saved_can_use_partial_vectors_p;
2807 151136 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2808 151136 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2809 151136 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2810 151136 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
2811 :
2812 151136 : if (loop_vinfo->scan_map)
2813 122 : loop_vinfo->scan_map->empty ();
2814 :
2815 151136 : goto start_over;
2816 : }
2817 :
2818 : /* Return true if vectorizing a loop using NEW_LOOP_VINFO appears
2819 : to be better than vectorizing it using OLD_LOOP_VINFO. Assume that
2820 : OLD_LOOP_VINFO is better unless something specifically indicates
2821 : otherwise.
2822 :
2823 : Note that this deliberately isn't a partial order. */
2824 :
2825 : static bool
2826 36217 : vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo,
2827 : loop_vec_info old_loop_vinfo)
2828 : {
2829 36217 : struct loop *loop = LOOP_VINFO_LOOP (new_loop_vinfo);
2830 36217 : gcc_assert (LOOP_VINFO_LOOP (old_loop_vinfo) == loop);
2831 :
2832 36217 : poly_int64 new_vf = LOOP_VINFO_VECT_FACTOR (new_loop_vinfo);
2833 36217 : poly_int64 old_vf = LOOP_VINFO_VECT_FACTOR (old_loop_vinfo);
2834 :
2835 : /* Always prefer a VF of loop->simdlen over any other VF. */
2836 36217 : if (loop->simdlen)
2837 : {
2838 0 : bool new_simdlen_p = known_eq (new_vf, loop->simdlen);
2839 0 : bool old_simdlen_p = known_eq (old_vf, loop->simdlen);
2840 0 : if (new_simdlen_p != old_simdlen_p)
2841 : return new_simdlen_p;
2842 : }
2843 :
2844 36217 : const auto *old_costs = old_loop_vinfo->vector_costs;
2845 36217 : const auto *new_costs = new_loop_vinfo->vector_costs;
2846 36217 : if (loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo))
2847 1573 : return new_costs->better_epilogue_loop_than_p (old_costs, main_loop);
2848 :
2849 34644 : return new_costs->better_main_loop_than_p (old_costs);
2850 : }
2851 :
2852 : /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO. Return
2853 : true if we should. */
2854 :
2855 : static bool
2856 36217 : vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
2857 : loop_vec_info old_loop_vinfo)
2858 : {
2859 36217 : if (!vect_better_loop_vinfo_p (new_loop_vinfo, old_loop_vinfo))
2860 : return false;
2861 :
2862 1487 : if (dump_enabled_p ())
2863 18 : dump_printf_loc (MSG_NOTE, vect_location,
2864 : "***** Preferring vector mode %s to vector mode %s\n",
2865 18 : GET_MODE_NAME (new_loop_vinfo->vector_mode),
2866 18 : GET_MODE_NAME (old_loop_vinfo->vector_mode));
2867 : return true;
2868 : }
2869 :
2870 : /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
2871 : not NULL. When MASKED_P is not -1 override the default
2872 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
2873 : Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
2874 : mode useful to analyze.
2875 : Return the loop_vinfo on success and wrapped null on failure. */
2876 :
2877 : static opt_loop_vec_info
2878 540265 : vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
2879 : const vect_loop_form_info *loop_form_info,
2880 : loop_vec_info orig_loop_vinfo,
2881 : const vector_modes &vector_modes, unsigned &mode_i,
2882 : int masked_p,
2883 : machine_mode &autodetected_vector_mode,
2884 : bool &fatal)
2885 : {
2886 540265 : loop_vec_info loop_vinfo
2887 540265 : = vect_create_loop_vinfo (loop, shared, loop_form_info, orig_loop_vinfo);
2888 :
2889 540265 : machine_mode vector_mode = vector_modes[mode_i];
2890 540265 : loop_vinfo->vector_mode = vector_mode;
2891 540265 : unsigned int suggested_unroll_factor = 1;
2892 540265 : bool single_lane_slp_done_for_suggested_uf = false;
2893 :
2894 : /* Run the main analysis. */
2895 540265 : opt_result res = vect_analyze_loop_2 (loop_vinfo, masked_p, fatal,
2896 : &suggested_unroll_factor,
2897 : single_lane_slp_done_for_suggested_uf);
2898 540265 : if (dump_enabled_p ())
2899 20965 : dump_printf_loc (MSG_NOTE, vect_location,
2900 : "***** Analysis %s with vector mode %s\n",
2901 20965 : res ? "succeeded" : "failed",
2902 20965 : GET_MODE_NAME (loop_vinfo->vector_mode));
2903 :
2904 540265 : auto user_unroll = LOOP_VINFO_LOOP (loop_vinfo)->unroll;
2905 540265 : if (res && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2906 : /* Check to see if the user wants to unroll or if the target wants to. */
2907 632366 : && (suggested_unroll_factor > 1 || user_unroll > 1))
2908 : {
2909 485 : if (suggested_unroll_factor == 1)
2910 : {
2911 66 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
2912 66 : suggested_unroll_factor = user_unroll / assumed_vf;
2913 66 : if (suggested_unroll_factor > 1)
2914 : {
2915 40 : if (dump_enabled_p ())
2916 20 : dump_printf_loc (MSG_NOTE, vect_location,
2917 : "setting unroll factor to %d based on user requested "
2918 : "unroll factor %d and suggested vectorization "
2919 : "factor: %d\n",
2920 : suggested_unroll_factor, user_unroll, assumed_vf);
2921 : }
2922 : }
2923 :
2924 485 : if (suggested_unroll_factor > 1)
2925 : {
2926 459 : if (dump_enabled_p ())
2927 62 : dump_printf_loc (MSG_NOTE, vect_location,
2928 : "***** Re-trying analysis for unrolling"
2929 : " with unroll factor %d and %s slp.\n",
2930 : suggested_unroll_factor,
2931 : single_lane_slp_done_for_suggested_uf
2932 : ? "single-lane" : "");
2933 459 : loop_vec_info unroll_vinfo
2934 459 : = vect_create_loop_vinfo (loop, shared, loop_form_info, NULL);
2935 459 : unroll_vinfo->vector_mode = vector_mode;
2936 459 : unroll_vinfo->suggested_unroll_factor = suggested_unroll_factor;
2937 459 : opt_result new_res
2938 459 : = vect_analyze_loop_2 (unroll_vinfo, masked_p, fatal, NULL,
2939 : single_lane_slp_done_for_suggested_uf);
2940 459 : if (new_res)
2941 : {
2942 400 : delete loop_vinfo;
2943 : loop_vinfo = unroll_vinfo;
2944 : }
2945 : else
2946 59 : delete unroll_vinfo;
2947 : }
2948 :
2949 : /* Record that we have honored a user unroll factor. */
2950 485 : LOOP_VINFO_USER_UNROLL (loop_vinfo) = user_unroll > 1;
2951 : }
2952 :
2953 : /* Remember the autodetected vector mode. */
2954 540265 : if (vector_mode == VOIDmode)
2955 215267 : autodetected_vector_mode = loop_vinfo->vector_mode;
2956 :
2957 : /* Advance mode_i, first skipping modes that would result in the
2958 : same analysis result. */
2959 2095847 : while (mode_i + 1 < vector_modes.length ()
2960 1392530 : && vect_chooses_same_modes_p (loop_vinfo,
2961 614739 : vector_modes[mode_i + 1]))
2962 : {
2963 237526 : if (dump_enabled_p ())
2964 12670 : dump_printf_loc (MSG_NOTE, vect_location,
2965 : "***** The result for vector mode %s would"
2966 : " be the same\n",
2967 12670 : GET_MODE_NAME (vector_modes[mode_i + 1]));
2968 237526 : mode_i += 1;
2969 : }
2970 540265 : if (mode_i + 1 < vector_modes.length ()
2971 917478 : && vect_chooses_same_modes_p (autodetected_vector_mode,
2972 377213 : vector_modes[mode_i + 1]))
2973 : {
2974 471 : if (dump_enabled_p ())
2975 23 : dump_printf_loc (MSG_NOTE, vect_location,
2976 : "***** Skipping vector mode %s, which would"
2977 : " repeat the analysis for %s\n",
2978 23 : GET_MODE_NAME (vector_modes[mode_i + 1]),
2979 23 : GET_MODE_NAME (autodetected_vector_mode));
2980 471 : mode_i += 1;
2981 : }
2982 540265 : mode_i++;
2983 :
2984 540265 : if (!res)
2985 : {
2986 439645 : delete loop_vinfo;
2987 439645 : if (fatal)
2988 42524 : gcc_checking_assert (orig_loop_vinfo == NULL);
2989 439645 : return opt_loop_vec_info::propagate_failure (res);
2990 : }
2991 :
2992 100620 : return opt_loop_vec_info::success (loop_vinfo);
2993 : }
2994 :
2995 : /* Function vect_analyze_loop.
2996 :
2997 : Apply a set of analyses on LOOP, and create a loop_vec_info struct
2998 : for it. The different analyses will record information in the
2999 : loop_vec_info struct. */
3000 : opt_loop_vec_info
3001 481919 : vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
3002 : vec_info_shared *shared)
3003 : {
3004 481919 : DUMP_VECT_SCOPE ("analyze_loop_nest");
3005 :
3006 481919 : if (loop_outer (loop)
3007 481919 : && loop_vec_info_for_loop (loop_outer (loop))
3008 482478 : && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
3009 559 : return opt_loop_vec_info::failure_at (vect_location,
3010 : "outer-loop already vectorized.\n");
3011 :
3012 481360 : if (!find_loop_nest (loop, &shared->loop_nest))
3013 23146 : return opt_loop_vec_info::failure_at
3014 23146 : (vect_location,
3015 : "not vectorized: loop nest containing two or more consecutive inner"
3016 : " loops cannot be vectorized\n");
3017 :
3018 : /* Analyze the loop form. */
3019 458214 : vect_loop_form_info loop_form_info;
3020 458214 : opt_result res = vect_analyze_loop_form (loop, loop_vectorized_call,
3021 : &loop_form_info);
3022 458214 : if (!res)
3023 : {
3024 179621 : if (dump_enabled_p ())
3025 1549 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3026 : "bad loop form.\n");
3027 179621 : return opt_loop_vec_info::propagate_failure (res);
3028 : }
3029 :
3030 278593 : res = vect_get_datarefs_in_loop (loop, loop_form_info.bbs,
3031 : &shared->datarefs);
3032 278593 : if (!res)
3033 : {
3034 63197 : if (dump_enabled_p ())
3035 1639 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3036 : "not vectorized: loop contains function "
3037 : "calls or data references that cannot "
3038 : "be analyzed\n");
3039 63197 : return opt_loop_vec_info::propagate_failure (res);
3040 : }
3041 215396 : shared->save_datarefs ();
3042 :
3043 215396 : res = vect_build_assumptions (loop, shared, &loop_form_info);
3044 215396 : if (!res)
3045 129 : return opt_loop_vec_info::propagate_failure (res);
3046 :
3047 215267 : auto_vector_modes vector_modes;
3048 : /* Autodetect first vector size we try. */
3049 215267 : vector_modes.safe_push (VOIDmode);
3050 215267 : unsigned int autovec_flags
3051 430534 : = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
3052 215267 : loop->simdlen != 0);
3053 215267 : bool pick_lowest_cost_p = ((autovec_flags & VECT_COMPARE_COSTS)
3054 215267 : && !unlimited_cost_model (loop));
3055 215267 : machine_mode autodetected_vector_mode = VOIDmode;
3056 215267 : opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
3057 215267 : unsigned int mode_i = 0;
3058 215267 : unsigned HOST_WIDE_INT simdlen = loop->simdlen;
3059 :
3060 : /* Keep track of the VF for each mode. Initialize all to 0 which indicates
3061 : a mode has not been analyzed. */
3062 215267 : auto_vec<poly_uint64, 8> cached_vf_per_mode;
3063 2177000 : for (unsigned i = 0; i < vector_modes.length (); ++i)
3064 873233 : cached_vf_per_mode.safe_push (0);
3065 :
3066 : /* First determine the main loop vectorization mode, either the first
3067 : one that works, starting with auto-detecting the vector mode and then
3068 : following the targets order of preference, or the one with the
3069 : lowest cost if pick_lowest_cost_p. */
3070 836273 : while (1)
3071 : {
3072 525770 : bool fatal;
3073 525770 : unsigned int last_mode_i = mode_i;
3074 : /* Set cached VF to -1 prior to analysis, which indicates a mode has
3075 : failed. */
3076 525770 : cached_vf_per_mode[last_mode_i] = -1;
3077 525770 : opt_loop_vec_info loop_vinfo
3078 525770 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3079 : NULL, vector_modes, mode_i, -1,
3080 : autodetected_vector_mode, fatal);
3081 525770 : if (fatal)
3082 : break;
3083 :
3084 483246 : if (loop_vinfo)
3085 : {
3086 : /* Analysis has been successful so update the VF value. The
3087 : VF should always be a multiple of unroll_factor and we want to
3088 : capture the original VF here. */
3089 92101 : cached_vf_per_mode[last_mode_i]
3090 92101 : = exact_div (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
3091 92101 : loop_vinfo->suggested_unroll_factor);
3092 : /* Once we hit the desired simdlen for the first time,
3093 : discard any previous attempts. */
3094 92101 : if (simdlen
3095 92101 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), simdlen))
3096 : {
3097 47 : delete first_loop_vinfo;
3098 92101 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3099 92101 : simdlen = 0;
3100 : }
3101 92054 : else if (pick_lowest_cost_p
3102 77738 : && first_loop_vinfo
3103 126698 : && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
3104 : {
3105 : /* Pick loop_vinfo over first_loop_vinfo. */
3106 1317 : delete first_loop_vinfo;
3107 1317 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3108 : }
3109 92101 : if (first_loop_vinfo == NULL)
3110 : first_loop_vinfo = loop_vinfo;
3111 : else
3112 : {
3113 33329 : delete loop_vinfo;
3114 33329 : loop_vinfo = opt_loop_vec_info::success (NULL);
3115 : }
3116 :
3117 : /* Commit to first_loop_vinfo if we have no reason to try
3118 : alternatives. */
3119 92101 : if (!simdlen && !pick_lowest_cost_p)
3120 : break;
3121 : }
3122 468892 : if (mode_i == vector_modes.length ()
3123 468892 : || autodetected_vector_mode == VOIDmode)
3124 : break;
3125 :
3126 : /* Try the next biggest vector size. */
3127 310503 : if (dump_enabled_p ())
3128 5227 : dump_printf_loc (MSG_NOTE, vect_location,
3129 : "***** Re-trying analysis with vector mode %s\n",
3130 5227 : GET_MODE_NAME (vector_modes[mode_i]));
3131 310503 : }
3132 215267 : if (!first_loop_vinfo)
3133 157817 : return opt_loop_vec_info::propagate_failure (res);
3134 :
3135 57450 : if (dump_enabled_p ())
3136 9935 : dump_printf_loc (MSG_NOTE, vect_location,
3137 : "***** Choosing vector mode %s\n",
3138 9935 : GET_MODE_NAME (first_loop_vinfo->vector_mode));
3139 :
3140 : /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
3141 : enabled, SIMDUID is not set, it is the innermost loop and we have
3142 : either already found the loop's SIMDLEN or there was no SIMDLEN to
3143 : begin with.
3144 : TODO: Enable epilogue vectorization for loops with SIMDUID set. */
3145 57450 : bool vect_epilogues = (!simdlen
3146 57448 : && loop->inner == NULL
3147 56887 : && param_vect_epilogues_nomask
3148 55745 : && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
3149 : /* No code motion support for multiple epilogues so for now
3150 : not supported when multiple exits. */
3151 27206 : && !LOOP_VINFO_EARLY_BREAKS (first_loop_vinfo)
3152 26686 : && !loop->simduid
3153 82719 : && loop_cost_model (loop) > VECT_COST_MODEL_VERY_CHEAP);
3154 57450 : if (!vect_epilogues)
3155 43991 : return first_loop_vinfo;
3156 :
3157 : /* Now analyze first_loop_vinfo for epilogue vectorization. */
3158 :
3159 : /* For epilogues start the analysis from the first mode. The motivation
3160 : behind starting from the beginning comes from cases where the VECTOR_MODES
3161 : array may contain length-agnostic and length-specific modes. Their
3162 : ordering is not guaranteed, so we could end up picking a mode for the main
3163 : loop that is after the epilogue's optimal mode. */
3164 13459 : int masked_p = -1;
3165 13459 : if (!unlimited_cost_model (loop)
3166 13459 : && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3167 : != VOIDmode))
3168 : {
3169 5 : vector_modes[0]
3170 5 : = first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3171 5 : cached_vf_per_mode[0] = 0;
3172 : }
3173 : else
3174 13454 : vector_modes[0] = autodetected_vector_mode;
3175 13459 : mode_i = 0;
3176 :
3177 13499 : bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
3178 13459 : || masked_p == 1);
3179 : if (supports_partial_vectors
3180 40 : && !partial_vectors_supported_p ()
3181 40 : && !LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (first_loop_vinfo))
3182 : supports_partial_vectors = false;
3183 13459 : poly_uint64 first_vinfo_vf = LOOP_VINFO_VECT_FACTOR (first_loop_vinfo);
3184 :
3185 13459 : loop_vec_info orig_loop_vinfo = first_loop_vinfo;
3186 13655 : do
3187 : {
3188 : /* Let the user override what the target suggests. */
3189 13557 : if (OPTION_SET_P (param_vect_partial_vector_usage))
3190 51 : masked_p = -1;
3191 :
3192 52427 : while (1)
3193 : {
3194 : /* If the target does not support partial vectors we can shorten the
3195 : number of modes to analyze for the epilogue as we know we can't
3196 : pick a mode that would lead to a VF at least as big as the
3197 : FIRST_VINFO_VF. */
3198 69721 : if (!supports_partial_vectors
3199 52427 : && maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
3200 : {
3201 24672 : mode_i++;
3202 49344 : if (mode_i == vector_modes.length ())
3203 : break;
3204 30554 : continue;
3205 : }
3206 : /* We would need an exhaustive search to find all modes we
3207 : skipped but that would lead to the same result as the
3208 : analysis it was skipped for and where we'd could check
3209 : cached_vf_per_mode against.
3210 : Check for the autodetected mode, which is the common
3211 : situation on x86 which does not perform cost comparison. */
3212 41015 : if (!supports_partial_vectors
3213 27717 : && maybe_ge (cached_vf_per_mode[0], first_vinfo_vf)
3214 54623 : && vect_chooses_same_modes_p (autodetected_vector_mode,
3215 26868 : vector_modes[mode_i]))
3216 : {
3217 13260 : mode_i++;
3218 26520 : if (mode_i == vector_modes.length ())
3219 : break;
3220 13260 : continue;
3221 : }
3222 :
3223 14495 : if (dump_enabled_p ())
3224 3599 : dump_printf_loc (MSG_NOTE, vect_location,
3225 : "***** Re-trying epilogue analysis with vector "
3226 3599 : "mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
3227 :
3228 14495 : bool fatal;
3229 14495 : opt_loop_vec_info loop_vinfo
3230 14495 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3231 : orig_loop_vinfo,
3232 : vector_modes, mode_i, masked_p,
3233 : autodetected_vector_mode, fatal);
3234 14495 : if (fatal)
3235 : break;
3236 :
3237 14495 : if (loop_vinfo)
3238 : {
3239 8519 : if (pick_lowest_cost_p
3240 5466 : && orig_loop_vinfo->epilogue_vinfo
3241 10092 : && vect_joust_loop_vinfos (loop_vinfo,
3242 1573 : orig_loop_vinfo->epilogue_vinfo))
3243 : {
3244 170 : gcc_assert (vect_epilogues);
3245 170 : delete orig_loop_vinfo->epilogue_vinfo;
3246 170 : orig_loop_vinfo->epilogue_vinfo = nullptr;
3247 : }
3248 8519 : if (!orig_loop_vinfo->epilogue_vinfo)
3249 7116 : orig_loop_vinfo->epilogue_vinfo = loop_vinfo;
3250 : else
3251 : {
3252 1403 : delete loop_vinfo;
3253 1403 : loop_vinfo = opt_loop_vec_info::success (NULL);
3254 : }
3255 :
3256 : /* For now only allow one epilogue loop, but allow
3257 : pick_lowest_cost_p to replace it, so commit to the
3258 : first epilogue if we have no reason to try alternatives. */
3259 8519 : if (!pick_lowest_cost_p)
3260 : break;
3261 : }
3262 :
3263 : /* Revert back to the default from the suggested preferred
3264 : epilogue vectorization mode. */
3265 11442 : masked_p = -1;
3266 22884 : if (mode_i == vector_modes.length ())
3267 : break;
3268 : }
3269 :
3270 13557 : orig_loop_vinfo = orig_loop_vinfo->epilogue_vinfo;
3271 13557 : if (!orig_loop_vinfo)
3272 : break;
3273 :
3274 : /* When we selected a first vectorized epilogue, see if the target
3275 : suggests to have another one. */
3276 6946 : masked_p = -1;
3277 6946 : if (!unlimited_cost_model (loop)
3278 3899 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
3279 10838 : && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3280 : != VOIDmode))
3281 : {
3282 196 : vector_modes[0]
3283 98 : = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3284 98 : cached_vf_per_mode[0] = 0;
3285 98 : mode_i = 0;
3286 : }
3287 : else
3288 : break;
3289 98 : }
3290 : while (1);
3291 :
3292 13459 : if (first_loop_vinfo->epilogue_vinfo)
3293 : {
3294 6855 : poly_uint64 lowest_th
3295 6855 : = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
3296 6855 : loop_vec_info epilog_vinfo = first_loop_vinfo->epilogue_vinfo;
3297 6946 : do
3298 : {
3299 6946 : poly_uint64 th = LOOP_VINFO_VERSIONING_THRESHOLD (epilog_vinfo);
3300 6946 : gcc_assert (!LOOP_REQUIRES_VERSIONING (epilog_vinfo)
3301 : || maybe_ne (lowest_th, 0U));
3302 : /* Keep track of the known smallest versioning threshold. */
3303 6946 : if (ordered_p (lowest_th, th))
3304 6946 : lowest_th = ordered_min (lowest_th, th);
3305 6946 : epilog_vinfo = epilog_vinfo->epilogue_vinfo;
3306 : }
3307 6946 : while (epilog_vinfo);
3308 6855 : LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
3309 6855 : if (dump_enabled_p ())
3310 1554 : dump_printf_loc (MSG_NOTE, vect_location,
3311 : "***** Choosing epilogue vector mode %s\n",
3312 1554 : GET_MODE_NAME
3313 : (first_loop_vinfo->epilogue_vinfo->vector_mode));
3314 : }
3315 :
3316 13459 : return first_loop_vinfo;
3317 458214 : }
3318 :
3319 : /* Return true if there is an in-order reduction function for CODE, storing
3320 : it in *REDUC_FN if so. */
3321 :
3322 : static bool
3323 5531 : fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
3324 : {
3325 : /* We support MINUS_EXPR by negating the operand. This also preserves an
3326 : initial -0.0 since -0.0 - 0.0 (neutral op for MINUS_EXPR) == -0.0 +
3327 : (-0.0) = -0.0. */
3328 5531 : if (code == PLUS_EXPR || code == MINUS_EXPR)
3329 : {
3330 4852 : *reduc_fn = IFN_FOLD_LEFT_PLUS;
3331 0 : return true;
3332 : }
3333 : return false;
3334 : }
3335 :
3336 : /* Function reduction_fn_for_scalar_code
3337 :
3338 : Input:
3339 : CODE - tree_code of a reduction operations.
3340 :
3341 : Output:
3342 : REDUC_FN - the corresponding internal function to be used to reduce the
3343 : vector of partial results into a single scalar result, or IFN_LAST
3344 : if the operation is a supported reduction operation, but does not have
3345 : such an internal function.
3346 :
3347 : Return FALSE if CODE currently cannot be vectorized as reduction. */
3348 :
3349 : bool
3350 2284022 : reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
3351 : {
3352 2284022 : if (code.is_tree_code ())
3353 2283964 : switch (tree_code (code))
3354 : {
3355 14211 : case MAX_EXPR:
3356 14211 : *reduc_fn = IFN_REDUC_MAX;
3357 14211 : return true;
3358 :
3359 49298 : case MIN_EXPR:
3360 49298 : *reduc_fn = IFN_REDUC_MIN;
3361 49298 : return true;
3362 :
3363 1248079 : case PLUS_EXPR:
3364 1248079 : *reduc_fn = IFN_REDUC_PLUS;
3365 1248079 : return true;
3366 :
3367 251006 : case BIT_AND_EXPR:
3368 251006 : *reduc_fn = IFN_REDUC_AND;
3369 251006 : return true;
3370 :
3371 270994 : case BIT_IOR_EXPR:
3372 270994 : *reduc_fn = IFN_REDUC_IOR;
3373 270994 : return true;
3374 :
3375 45295 : case BIT_XOR_EXPR:
3376 45295 : *reduc_fn = IFN_REDUC_XOR;
3377 45295 : return true;
3378 :
3379 405081 : case MULT_EXPR:
3380 405081 : case MINUS_EXPR:
3381 405081 : *reduc_fn = IFN_LAST;
3382 405081 : return true;
3383 :
3384 : default:
3385 : return false;
3386 : }
3387 : else
3388 58 : switch (combined_fn (code))
3389 : {
3390 34 : CASE_CFN_FMAX:
3391 34 : *reduc_fn = IFN_REDUC_FMAX;
3392 34 : return true;
3393 :
3394 24 : CASE_CFN_FMIN:
3395 24 : *reduc_fn = IFN_REDUC_FMIN;
3396 24 : return true;
3397 :
3398 : default:
3399 : return false;
3400 : }
3401 : }
3402 :
3403 : /* Set *SBOOL_FN to the corresponding function working on vector masks
3404 : for REDUC_FN. Return true if that exists, false otherwise. */
3405 :
3406 : static bool
3407 0 : sbool_reduction_fn_for_fn (internal_fn reduc_fn, internal_fn *sbool_fn)
3408 : {
3409 0 : switch (reduc_fn)
3410 : {
3411 0 : case IFN_REDUC_AND:
3412 0 : *sbool_fn = IFN_REDUC_SBOOL_AND;
3413 0 : return true;
3414 0 : case IFN_REDUC_IOR:
3415 0 : *sbool_fn = IFN_REDUC_SBOOL_IOR;
3416 0 : return true;
3417 0 : case IFN_REDUC_XOR:
3418 0 : *sbool_fn = IFN_REDUC_SBOOL_XOR;
3419 0 : return true;
3420 : default:
3421 : return false;
3422 : }
3423 : }
3424 :
3425 : /* If there is a neutral value X such that a reduction would not be affected
3426 : by the introduction of additional X elements, return that X, otherwise
3427 : return null. CODE is the code of the reduction and SCALAR_TYPE is type
3428 : of the scalar elements. If the reduction has just a single initial value
3429 : then INITIAL_VALUE is that value, otherwise it is null.
3430 : If AS_INITIAL is TRUE the value is supposed to be used as initial value.
3431 : In that case no signed zero is returned. */
3432 :
3433 : tree
3434 75628 : neutral_op_for_reduction (tree scalar_type, code_helper code,
3435 : tree initial_value, bool as_initial)
3436 : {
3437 75628 : if (code.is_tree_code ())
3438 75570 : switch (tree_code (code))
3439 : {
3440 14015 : case DOT_PROD_EXPR:
3441 14015 : case SAD_EXPR:
3442 14015 : case MINUS_EXPR:
3443 14015 : case BIT_IOR_EXPR:
3444 14015 : case BIT_XOR_EXPR:
3445 14015 : return build_zero_cst (scalar_type);
3446 55250 : case WIDEN_SUM_EXPR:
3447 55250 : case PLUS_EXPR:
3448 55250 : if (!as_initial && HONOR_SIGNED_ZEROS (scalar_type))
3449 104 : return build_real (scalar_type, dconstm0);
3450 : else
3451 55146 : return build_zero_cst (scalar_type);
3452 :
3453 2252 : case MULT_EXPR:
3454 2252 : return build_one_cst (scalar_type);
3455 :
3456 1573 : case BIT_AND_EXPR:
3457 1573 : return build_all_ones_cst (scalar_type);
3458 :
3459 : case MAX_EXPR:
3460 : case MIN_EXPR:
3461 : return initial_value;
3462 :
3463 447 : default:
3464 447 : return NULL_TREE;
3465 : }
3466 : else
3467 58 : switch (combined_fn (code))
3468 : {
3469 : CASE_CFN_FMIN:
3470 : CASE_CFN_FMAX:
3471 : return initial_value;
3472 :
3473 0 : default:
3474 0 : return NULL_TREE;
3475 : }
3476 : }
3477 :
3478 : /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
3479 : STMT is printed with a message MSG. */
3480 :
3481 : static void
3482 581 : report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
3483 : {
3484 581 : dump_printf_loc (msg_type, vect_location, "%s%G", msg, stmt);
3485 581 : }
3486 :
3487 : /* Return true if we need an in-order reduction for operation CODE
3488 : on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
3489 : overflow must wrap. */
3490 :
3491 : bool
3492 363249 : needs_fold_left_reduction_p (tree type, code_helper code)
3493 : {
3494 : /* CHECKME: check for !flag_finite_math_only too? */
3495 363249 : if (SCALAR_FLOAT_TYPE_P (type))
3496 : {
3497 97595 : if (code.is_tree_code ())
3498 97541 : switch (tree_code (code))
3499 : {
3500 : case MIN_EXPR:
3501 : case MAX_EXPR:
3502 : return false;
3503 :
3504 97004 : default:
3505 97004 : return !flag_associative_math;
3506 : }
3507 : else
3508 54 : switch (combined_fn (code))
3509 : {
3510 : CASE_CFN_FMIN:
3511 : CASE_CFN_FMAX:
3512 : return false;
3513 :
3514 2 : default:
3515 2 : return !flag_associative_math;
3516 : }
3517 : }
3518 :
3519 265654 : if (INTEGRAL_TYPE_P (type))
3520 265544 : return (!code.is_tree_code ()
3521 265544 : || !operation_no_trapping_overflow (type, tree_code (code)));
3522 :
3523 110 : if (SAT_FIXED_POINT_TYPE_P (type))
3524 : return true;
3525 :
3526 : return false;
3527 : }
3528 :
3529 : /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
3530 : has a handled computation expression. Store the main reduction
3531 : operation in *CODE. */
3532 :
3533 : static bool
3534 104097 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3535 : tree loop_arg, code_helper *code,
3536 : vec<std::pair<ssa_op_iter, use_operand_p> > &path,
3537 : bool inner_loop_of_double_reduc)
3538 : {
3539 104097 : auto_bitmap visited;
3540 104097 : tree lookfor = PHI_RESULT (phi);
3541 104097 : ssa_op_iter curri;
3542 104097 : use_operand_p curr = op_iter_init_phiuse (&curri, phi, SSA_OP_USE);
3543 216589 : while (USE_FROM_PTR (curr) != loop_arg)
3544 8395 : curr = op_iter_next_use (&curri);
3545 104097 : curri.i = curri.numops;
3546 966853 : do
3547 : {
3548 966853 : path.safe_push (std::make_pair (curri, curr));
3549 966853 : tree use = USE_FROM_PTR (curr);
3550 966853 : if (use == lookfor)
3551 : break;
3552 863467 : gimple *def = SSA_NAME_DEF_STMT (use);
3553 863467 : if (gimple_nop_p (def)
3554 863467 : || ! flow_bb_inside_loop_p (loop, gimple_bb (def)))
3555 : {
3556 243028 : pop:
3557 726051 : do
3558 : {
3559 726051 : std::pair<ssa_op_iter, use_operand_p> x = path.pop ();
3560 726051 : curri = x.first;
3561 726051 : curr = x.second;
3562 794769 : do
3563 794769 : curr = op_iter_next_use (&curri);
3564 : /* Skip already visited or non-SSA operands (from iterating
3565 : over PHI args). */
3566 : while (curr != NULL_USE_OPERAND_P
3567 1105804 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3568 274402 : || ! bitmap_set_bit (visited,
3569 274402 : SSA_NAME_VERSION
3570 : (USE_FROM_PTR (curr)))));
3571 : }
3572 1452102 : while (curr == NULL_USE_OPERAND_P && ! path.is_empty ());
3573 243028 : if (curr == NULL_USE_OPERAND_P)
3574 : break;
3575 : }
3576 : else
3577 : {
3578 726578 : if (gimple_code (def) == GIMPLE_PHI)
3579 74109 : curr = op_iter_init_phiuse (&curri, as_a <gphi *>(def), SSA_OP_USE);
3580 : else
3581 652469 : curr = op_iter_init_use (&curri, def, SSA_OP_USE);
3582 : while (curr != NULL_USE_OPERAND_P
3583 868298 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3584 756424 : || ! bitmap_set_bit (visited,
3585 756424 : SSA_NAME_VERSION
3586 : (USE_FROM_PTR (curr)))))
3587 141720 : curr = op_iter_next_use (&curri);
3588 726578 : if (curr == NULL_USE_OPERAND_P)
3589 106139 : goto pop;
3590 : }
3591 : }
3592 : while (1);
3593 104097 : if (dump_file && (dump_flags & TDF_DETAILS))
3594 : {
3595 4268 : dump_printf_loc (MSG_NOTE, loc, "reduction path: ");
3596 4268 : unsigned i;
3597 4268 : std::pair<ssa_op_iter, use_operand_p> *x;
3598 14545 : FOR_EACH_VEC_ELT (path, i, x)
3599 10277 : dump_printf (MSG_NOTE, "%T ", USE_FROM_PTR (x->second));
3600 4268 : dump_printf (MSG_NOTE, "\n");
3601 : }
3602 :
3603 : /* Check whether the reduction path detected is valid. */
3604 104097 : bool fail = path.length () == 0;
3605 104097 : bool neg = false;
3606 104097 : int sign = -1;
3607 104097 : *code = ERROR_MARK;
3608 223869 : for (unsigned i = 1; i < path.length (); ++i)
3609 : {
3610 126831 : gimple *use_stmt = USE_STMT (path[i].second);
3611 126831 : gimple_match_op op;
3612 126831 : if (!gimple_extract_op (use_stmt, &op))
3613 : {
3614 : fail = true;
3615 7059 : break;
3616 : }
3617 125915 : unsigned int opi = op.num_ops;
3618 125915 : if (gassign *assign = dyn_cast<gassign *> (use_stmt))
3619 : {
3620 : /* The following make sure we can compute the operand index
3621 : easily plus it mostly disallows chaining via COND_EXPR condition
3622 : operands. */
3623 194860 : for (opi = 0; opi < op.num_ops; ++opi)
3624 193812 : if (gimple_assign_rhs1_ptr (assign) + opi == path[i].second->use)
3625 : break;
3626 : }
3627 6244 : else if (gcall *call = dyn_cast<gcall *> (use_stmt))
3628 : {
3629 12497 : for (opi = 0; opi < op.num_ops; ++opi)
3630 12497 : if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
3631 : break;
3632 : }
3633 125915 : if (opi == op.num_ops)
3634 : {
3635 : fail = true;
3636 : break;
3637 : }
3638 124867 : op.code = canonicalize_code (op.code, op.type);
3639 124867 : if (op.code == MINUS_EXPR)
3640 : {
3641 5691 : op.code = PLUS_EXPR;
3642 : /* Track whether we negate the reduction value each iteration. */
3643 5691 : if (op.ops[1] == op.ops[opi])
3644 34 : neg = ! neg;
3645 : }
3646 119176 : else if (op.code == IFN_COND_SUB)
3647 : {
3648 9 : op.code = IFN_COND_ADD;
3649 : /* Track whether we negate the reduction value each iteration. */
3650 9 : if (op.ops[2] == op.ops[opi])
3651 0 : neg = ! neg;
3652 : }
3653 : /* For an FMA the reduction code is the PLUS if the addition chain
3654 : is the reduction. */
3655 119167 : else if (op.code == IFN_FMA && opi == 2)
3656 35 : op.code = PLUS_EXPR;
3657 124867 : if (CONVERT_EXPR_CODE_P (op.code)
3658 124867 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
3659 : ;
3660 119241 : else if (*code == ERROR_MARK)
3661 : {
3662 101532 : *code = op.code;
3663 101532 : sign = TYPE_SIGN (op.type);
3664 : }
3665 17709 : else if (op.code != *code)
3666 : {
3667 : fail = true;
3668 : break;
3669 : }
3670 16149 : else if ((op.code == MIN_EXPR
3671 15993 : || op.code == MAX_EXPR)
3672 16162 : && sign != TYPE_SIGN (op.type))
3673 : {
3674 : fail = true;
3675 : break;
3676 : }
3677 : /* Check there's only a single stmt the op is used on. For the
3678 : not value-changing tail and the last stmt allow out-of-loop uses,
3679 : but not when this is the inner loop of a double reduction.
3680 : ??? We could relax this and handle arbitrary live stmts by
3681 : forcing a scalar epilogue for example. */
3682 123304 : imm_use_iterator imm_iter;
3683 123304 : use_operand_p use_p;
3684 123304 : gimple *op_use_stmt;
3685 123304 : unsigned cnt = 0;
3686 129511 : bool cond_fn_p = op.code.is_internal_fn ()
3687 6207 : && (conditional_internal_fn_code (internal_fn (op.code))
3688 123304 : != ERROR_MARK);
3689 :
3690 295418 : FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
3691 : {
3692 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we should
3693 : have op1 twice (once as definition, once as else) in the same
3694 : operation. Enforce this. */
3695 172114 : if (cond_fn_p && op_use_stmt == use_stmt)
3696 : {
3697 6121 : gcall *call = as_a<gcall *> (use_stmt);
3698 6121 : unsigned else_pos
3699 6121 : = internal_fn_else_index (internal_fn (op.code));
3700 6121 : if (gimple_call_arg (call, else_pos) != op.ops[opi])
3701 : {
3702 : fail = true;
3703 : break;
3704 : }
3705 30605 : for (unsigned int j = 0; j < gimple_call_num_args (call); ++j)
3706 : {
3707 24484 : if (j == else_pos)
3708 6121 : continue;
3709 18363 : if (gimple_call_arg (call, j) == op.ops[opi])
3710 6121 : cnt++;
3711 : }
3712 : }
3713 165993 : else if (!is_gimple_debug (op_use_stmt)
3714 165993 : && ((*code != ERROR_MARK || inner_loop_of_double_reduc)
3715 2838 : || flow_bb_inside_loop_p (loop,
3716 2838 : gimple_bb (op_use_stmt))))
3717 241577 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
3718 120793 : cnt++;
3719 123304 : }
3720 :
3721 123304 : if (cnt != 1)
3722 : {
3723 : fail = true;
3724 : break;
3725 : }
3726 : }
3727 111878 : return ! fail && ! neg && *code != ERROR_MARK;
3728 104097 : }
3729 :
3730 : bool
3731 21 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3732 : tree loop_arg, enum tree_code code)
3733 : {
3734 21 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3735 21 : code_helper code_;
3736 21 : return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path, false)
3737 21 : && code_ == code);
3738 21 : }
3739 :
3740 :
3741 :
3742 : /* Function vect_is_simple_reduction
3743 :
3744 : (1) Detect a cross-iteration def-use cycle that represents a simple
3745 : reduction computation. We look for the following pattern:
3746 :
3747 : loop_header:
3748 : a1 = phi < a0, a2 >
3749 : a3 = ...
3750 : a2 = operation (a3, a1)
3751 :
3752 : or
3753 :
3754 : a3 = ...
3755 : loop_header:
3756 : a1 = phi < a0, a2 >
3757 : a2 = operation (a3, a1)
3758 :
3759 : such that:
3760 : 1. operation is commutative and associative and it is safe to
3761 : change the order of the computation
3762 : 2. no uses for a2 in the loop (a2 is used out of the loop)
3763 : 3. no uses of a1 in the loop besides the reduction operation
3764 : 4. no uses of a1 outside the loop.
3765 :
3766 : Conditions 1,4 are tested here.
3767 : Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
3768 :
3769 : (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
3770 : nested cycles.
3771 :
3772 : (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
3773 : reductions:
3774 :
3775 : a1 = phi < a0, a2 >
3776 : inner loop (def of a3)
3777 : a2 = phi < a3 >
3778 :
3779 : (4) Detect condition expressions, ie:
3780 : for (int i = 0; i < N; i++)
3781 : if (a[i] < val)
3782 : ret_val = a[i];
3783 :
3784 : */
3785 :
3786 : static stmt_vec_info
3787 170964 : vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
3788 : gphi **double_reduc)
3789 : {
3790 170964 : gphi *phi = as_a <gphi *> (phi_info->stmt);
3791 170964 : gimple *phi_use_stmt = NULL;
3792 170964 : imm_use_iterator imm_iter;
3793 170964 : use_operand_p use_p;
3794 :
3795 : /* When double_reduc is NULL we are testing the inner loop of a
3796 : double reduction. */
3797 170964 : bool inner_loop_of_double_reduc = double_reduc == NULL;
3798 170964 : if (double_reduc)
3799 169835 : *double_reduc = NULL;
3800 170964 : STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION;
3801 :
3802 170964 : tree phi_name = PHI_RESULT (phi);
3803 : /* ??? If there are no uses of the PHI result the inner loop reduction
3804 : won't be detected as possibly double-reduction by vectorizable_reduction
3805 : because that tries to walk the PHI arg from the preheader edge which
3806 : can be constant. See PR60382. */
3807 170964 : if (has_zero_uses (phi_name))
3808 : return NULL;
3809 170832 : class loop *loop = (gimple_bb (phi))->loop_father;
3810 170832 : unsigned nphi_def_loop_uses = 0;
3811 476125 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, phi_name)
3812 : {
3813 317796 : gimple *use_stmt = USE_STMT (use_p);
3814 317796 : if (is_gimple_debug (use_stmt))
3815 87173 : continue;
3816 :
3817 230623 : if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3818 : {
3819 12503 : if (dump_enabled_p ())
3820 31 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3821 : "intermediate value used outside loop.\n");
3822 :
3823 12503 : return NULL;
3824 : }
3825 :
3826 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
3827 : op1 twice (once as definition, once as else) in the same operation.
3828 : Only count it as one. */
3829 218120 : if (use_stmt != phi_use_stmt)
3830 : {
3831 211639 : nphi_def_loop_uses++;
3832 211639 : phi_use_stmt = use_stmt;
3833 : }
3834 12503 : }
3835 :
3836 158329 : tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
3837 158329 : if (TREE_CODE (latch_def) != SSA_NAME)
3838 : {
3839 1533 : if (dump_enabled_p ())
3840 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3841 : "reduction: not ssa_name: %T\n", latch_def);
3842 : return NULL;
3843 : }
3844 :
3845 156796 : stmt_vec_info def_stmt_info = loop_info->lookup_def (latch_def);
3846 156796 : if (!def_stmt_info
3847 156796 : || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt_info->stmt)))
3848 : return NULL;
3849 :
3850 156618 : bool nested_in_vect_loop
3851 156618 : = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info), loop);
3852 156618 : unsigned nlatch_def_loop_uses = 0;
3853 156618 : auto_vec<gphi *, 3> lcphis;
3854 618539 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, latch_def)
3855 : {
3856 461921 : gimple *use_stmt = USE_STMT (use_p);
3857 461921 : if (is_gimple_debug (use_stmt))
3858 144143 : continue;
3859 317778 : if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3860 199322 : nlatch_def_loop_uses++;
3861 : else
3862 : /* We can have more than one loop-closed PHI. */
3863 118456 : lcphis.safe_push (as_a <gphi *> (use_stmt));
3864 156618 : }
3865 :
3866 : /* If we are vectorizing an inner reduction we are executing that
3867 : in the original order only in case we are not dealing with a
3868 : double reduction. */
3869 156618 : if (nested_in_vect_loop && !inner_loop_of_double_reduc)
3870 : {
3871 2316 : if (dump_enabled_p ())
3872 437 : report_vect_op (MSG_NOTE, def_stmt_info->stmt,
3873 : "detected nested cycle: ");
3874 : return def_stmt_info;
3875 : }
3876 :
3877 : /* When the inner loop of a double reduction ends up with more than
3878 : one loop-closed PHI we have failed to classify alternate such
3879 : PHIs as double reduction, leading to wrong code. See PR103237. */
3880 155419 : if (inner_loop_of_double_reduc && lcphis.length () != 1)
3881 : {
3882 1 : if (dump_enabled_p ())
3883 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3884 : "unhandle double reduction\n");
3885 : return NULL;
3886 : }
3887 :
3888 : /* If this isn't a nested cycle or if the nested cycle reduction value
3889 : is used outside of the inner loop we cannot handle uses of the reduction
3890 : value. */
3891 154301 : if (nlatch_def_loop_uses > 1 || nphi_def_loop_uses > 1)
3892 : {
3893 48840 : if (dump_enabled_p ())
3894 412 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3895 : "reduction used in loop.\n");
3896 : return NULL;
3897 : }
3898 :
3899 : /* If DEF_STMT is a phi node itself, we expect it to have a single argument
3900 : defined in the inner loop. */
3901 105461 : if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt))
3902 : {
3903 1385 : tree op1 = PHI_ARG_DEF (def_stmt, 0);
3904 1385 : if (gimple_phi_num_args (def_stmt) != 1
3905 1385 : || TREE_CODE (op1) != SSA_NAME)
3906 : {
3907 92 : if (dump_enabled_p ())
3908 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3909 : "unsupported phi node definition.\n");
3910 :
3911 : return NULL;
3912 : }
3913 :
3914 : /* Verify there is an inner cycle composed of the PHI phi_use_stmt
3915 : and the latch definition op1. */
3916 1293 : gimple *def1 = SSA_NAME_DEF_STMT (op1);
3917 1293 : if (gimple_bb (def1)
3918 1293 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
3919 1293 : && loop->inner
3920 1239 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
3921 1239 : && (is_gimple_assign (def1) || is_gimple_call (def1))
3922 1230 : && is_a <gphi *> (phi_use_stmt)
3923 1218 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt))
3924 1218 : && (op1 == PHI_ARG_DEF_FROM_EDGE (phi_use_stmt,
3925 : loop_latch_edge (loop->inner)))
3926 159040 : && lcphis.length () == 1)
3927 : {
3928 1129 : if (dump_enabled_p ())
3929 144 : report_vect_op (MSG_NOTE, def_stmt,
3930 : "detected double reduction: ");
3931 :
3932 1129 : *double_reduc = as_a <gphi *> (phi_use_stmt);
3933 1129 : return def_stmt_info;
3934 : }
3935 :
3936 : return NULL;
3937 : }
3938 :
3939 : /* Look for the expression computing latch_def from then loop PHI result. */
3940 104076 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3941 104076 : code_helper code;
3942 104076 : if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
3943 : path, inner_loop_of_double_reduc))
3944 : {
3945 96295 : STMT_VINFO_REDUC_CODE (phi_info) = code;
3946 96295 : if (code == COND_EXPR && !nested_in_vect_loop)
3947 8512 : STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION;
3948 :
3949 : /* Fill in STMT_VINFO_REDUC_IDX. */
3950 96295 : unsigned i;
3951 310440 : for (i = path.length () - 1; i >= 1; --i)
3952 : {
3953 117850 : gimple *stmt = USE_STMT (path[i].second);
3954 117850 : stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
3955 117850 : gimple_match_op op;
3956 117850 : if (!gimple_extract_op (stmt, &op))
3957 0 : gcc_unreachable ();
3958 117850 : if (gassign *assign = dyn_cast<gassign *> (stmt))
3959 111630 : STMT_VINFO_REDUC_IDX (stmt_info)
3960 111630 : = path[i].second->use - gimple_assign_rhs1_ptr (assign);
3961 : else
3962 : {
3963 6220 : gcall *call = as_a<gcall *> (stmt);
3964 6220 : STMT_VINFO_REDUC_IDX (stmt_info)
3965 6220 : = path[i].second->use - gimple_call_arg_ptr (call, 0);
3966 : }
3967 : }
3968 96295 : if (dump_enabled_p ())
3969 4250 : dump_printf_loc (MSG_NOTE, vect_location,
3970 : "reduction: detected reduction\n");
3971 :
3972 : return def_stmt_info;
3973 : }
3974 :
3975 7781 : if (dump_enabled_p ())
3976 99 : dump_printf_loc (MSG_NOTE, vect_location,
3977 : "reduction: unknown pattern\n");
3978 :
3979 : return NULL;
3980 260694 : }
3981 :
3982 : /* Estimate the number of peeled epilogue iterations for LOOP_VINFO.
3983 : PEEL_ITERS_PROLOGUE is the number of peeled prologue iterations,
3984 : or -1 if not known. */
3985 :
3986 : static int
3987 517299 : vect_get_peel_iters_epilogue (loop_vec_info loop_vinfo, int peel_iters_prologue)
3988 : {
3989 517299 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
3990 517299 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) || peel_iters_prologue == -1)
3991 : {
3992 216099 : if (dump_enabled_p ())
3993 4345 : dump_printf_loc (MSG_NOTE, vect_location,
3994 : "cost model: epilogue peel iters set to vf/2 "
3995 : "because loop iterations are unknown .\n");
3996 216099 : return assumed_vf / 2;
3997 : }
3998 : else
3999 : {
4000 301200 : int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
4001 301200 : peel_iters_prologue = MIN (niters, peel_iters_prologue);
4002 301200 : int peel_iters_epilogue = (niters - peel_iters_prologue) % assumed_vf;
4003 : /* If we need to peel for gaps, but no peeling is required, we have to
4004 : peel VF iterations. */
4005 301200 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) && !peel_iters_epilogue)
4006 517299 : peel_iters_epilogue = assumed_vf;
4007 : return peel_iters_epilogue;
4008 : }
4009 : }
4010 :
4011 : /* Calculate cost of peeling the scalar loop PEEL_ITERS_PROLOGUE times for
4012 : a prologue and the corresponding times for the epilogue. */
4013 : int
4014 388573 : vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue)
4015 : {
4016 388573 : int retval = 0;
4017 :
4018 388573 : int peel_iters_epilogue
4019 388573 : = vect_get_peel_iters_epilogue (loop_vinfo, peel_iters_prologue);
4020 :
4021 388573 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
4022 : {
4023 : /* If peeled iterations are known but number of scalar loop
4024 : iterations are unknown, count a taken branch per peeled loop. */
4025 148087 : if (peel_iters_prologue > 0)
4026 90865 : retval = builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
4027 148087 : if (peel_iters_epilogue > 0)
4028 147974 : retval += builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
4029 : }
4030 :
4031 777146 : retval += ((peel_iters_prologue + peel_iters_epilogue)
4032 388573 : * loop_vinfo->scalar_costs->body_cost ());
4033 777146 : retval += (((peel_iters_prologue != 0) + (peel_iters_epilogue != 0))
4034 388573 : * loop_vinfo->scalar_costs->outside_cost ());
4035 :
4036 388573 : return retval;
4037 : }
4038 :
4039 : /* Return the count for SI in PEEL_ITERS scalar iterations, including the
4040 : inner-loop frequency. */
4041 :
4042 : static int
4043 317935 : vect_scalar_peeling_stmt_count (loop_vec_info loop_vinfo,
4044 : const stmt_info_for_cost &si, int peel_iters)
4045 : {
4046 317935 : int count = si.count;
4047 317935 : if (si.stmt_info && stmt_in_inner_loop_p (loop_vinfo, si.stmt_info))
4048 349 : count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
4049 317935 : return count * peel_iters;
4050 : }
4051 :
4052 : /* Function vect_estimate_min_profitable_iters
4053 :
4054 : Return the number of iterations required for the vector version of the
4055 : loop to be profitable relative to the cost of the scalar version of the
4056 : loop.
4057 :
4058 : *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
4059 : of iterations for vectorization. -1 value means loop vectorization
4060 : is not profitable. This returned value may be used for dynamic
4061 : profitability check.
4062 :
4063 : *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
4064 : for static check against estimated number of iterations. */
4065 :
4066 : static void
4067 146456 : vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
4068 : int *ret_min_profitable_niters,
4069 : int *ret_min_profitable_estimate,
4070 : unsigned *suggested_unroll_factor)
4071 : {
4072 146456 : int min_profitable_iters;
4073 146456 : int min_profitable_estimate;
4074 146456 : int peel_iters_prologue;
4075 146456 : int peel_iters_epilogue;
4076 146456 : unsigned vec_inside_cost = 0;
4077 146456 : int vec_outside_cost = 0;
4078 146456 : unsigned vec_prologue_cost = 0;
4079 146456 : unsigned vec_epilogue_cost = 0;
4080 146456 : int scalar_single_iter_cost = 0;
4081 146456 : int scalar_outside_cost = 0;
4082 146456 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
4083 146456 : int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
4084 146456 : vector_costs *target_cost_data = loop_vinfo->vector_costs;
4085 :
4086 : /* Cost model disabled. */
4087 146456 : if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
4088 : {
4089 17420 : if (dump_enabled_p ())
4090 11105 : dump_printf_loc (MSG_NOTE, vect_location, "cost model disabled.\n");
4091 17420 : *ret_min_profitable_niters = 0;
4092 17420 : *ret_min_profitable_estimate = 0;
4093 17420 : return;
4094 : }
4095 :
4096 : /* Requires loop versioning tests to handle misalignment. */
4097 129036 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
4098 : {
4099 : /* FIXME: Make cost depend on complexity of individual check. */
4100 18 : unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
4101 18 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4102 18 : if (dump_enabled_p ())
4103 2 : dump_printf (MSG_NOTE,
4104 : "cost model: Adding cost of checks for loop "
4105 : "versioning to treat misalignment.\n");
4106 : }
4107 :
4108 : /* Requires loop versioning with alias checks. */
4109 129036 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
4110 : {
4111 : /* FIXME: Make cost depend on complexity of individual check. */
4112 7297 : unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
4113 7297 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4114 7297 : len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
4115 4 : if (len)
4116 : /* Count LEN - 1 ANDs and LEN comparisons. */
4117 4 : (void) add_stmt_cost (target_cost_data, len * 2 - 1,
4118 : scalar_stmt, vect_prologue);
4119 7297 : len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
4120 1275 : if (len)
4121 : {
4122 : /* Count LEN - 1 ANDs and LEN comparisons. */
4123 1275 : unsigned int nstmts = len * 2 - 1;
4124 : /* +1 for each bias that needs adding. */
4125 2550 : for (unsigned int i = 0; i < len; ++i)
4126 1275 : if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
4127 154 : nstmts += 1;
4128 1275 : (void) add_stmt_cost (target_cost_data, nstmts,
4129 : scalar_stmt, vect_prologue);
4130 : }
4131 7297 : if (dump_enabled_p ())
4132 32 : dump_printf (MSG_NOTE,
4133 : "cost model: Adding cost of checks for loop "
4134 : "versioning aliasing.\n");
4135 : }
4136 :
4137 : /* Requires loop versioning with niter checks. */
4138 129036 : if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
4139 : {
4140 908 : tree assum = LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo);
4141 :
4142 : /* Count all involved statements in the assumption.
4143 : ??? This might over-count in case we CSE later. */
4144 6180 : auto count_stmts = [] (tree *t, int *walk, void *nstmts) -> tree
4145 : {
4146 5272 : if (!EXPR_P (*t) || is_gimple_val (*t))
4147 2619 : *walk = 0;
4148 : else
4149 2653 : (*(int *) nstmts)++;
4150 5272 : return NULL_TREE;
4151 : };
4152 :
4153 908 : int nstmts = 0;
4154 908 : walk_tree_without_duplicates (&assum, count_stmts, &nstmts);
4155 :
4156 908 : (void) add_stmt_cost (target_cost_data, nstmts, scalar_stmt,
4157 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4158 908 : if (dump_enabled_p ())
4159 1 : dump_printf (MSG_NOTE,
4160 : "cost model: Adding cost of checks for loop "
4161 : "versioning niters.\n");
4162 : }
4163 :
4164 129036 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4165 8141 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4166 : vect_prologue);
4167 :
4168 : /* Count statements in scalar loop. Using this as scalar cost for a single
4169 : iteration for now.
4170 :
4171 : TODO: Add outer loop support.
4172 :
4173 : TODO: Consider assigning different costs to different scalar
4174 : statements. */
4175 :
4176 129036 : scalar_single_iter_cost = loop_vinfo->scalar_costs->total_cost ();
4177 :
4178 : /* Add additional cost for the peeled instructions in prologue and epilogue
4179 : loop. (For fully-masked loops there will be no peeling.)
4180 :
4181 : FORNOW: If we don't know the value of peel_iters for prologue or epilogue
4182 : at compile-time - we assume it's vf/2 (the worst would be vf-1).
4183 :
4184 : TODO: Build an expression that represents peel_iters for prologue and
4185 : epilogue to be used in a run-time test. */
4186 :
4187 129036 : bool prologue_need_br_taken_cost = false;
4188 129036 : bool prologue_need_br_not_taken_cost = false;
4189 :
4190 : /* Calculate peel_iters_prologue. */
4191 129036 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
4192 : peel_iters_prologue = 0;
4193 129036 : else if (npeel < 0)
4194 : {
4195 282 : peel_iters_prologue = assumed_vf / 2;
4196 282 : if (dump_enabled_p ())
4197 11 : dump_printf (MSG_NOTE, "cost model: "
4198 : "prologue peel iters set to vf/2.\n");
4199 :
4200 : /* If peeled iterations are unknown, count a taken branch and a not taken
4201 : branch per peeled loop. Even if scalar loop iterations are known,
4202 : vector iterations are not known since peeled prologue iterations are
4203 : not known. Hence guards remain the same. */
4204 : prologue_need_br_taken_cost = true;
4205 : prologue_need_br_not_taken_cost = true;
4206 : }
4207 : else
4208 : {
4209 128754 : peel_iters_prologue = npeel;
4210 128754 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_prologue > 0)
4211 : /* If peeled iterations are known but number of scalar loop
4212 : iterations are unknown, count a taken branch per peeled loop. */
4213 129036 : prologue_need_br_taken_cost = true;
4214 : }
4215 :
4216 129036 : bool epilogue_need_br_taken_cost = false;
4217 129036 : bool epilogue_need_br_not_taken_cost = false;
4218 :
4219 : /* Calculate peel_iters_epilogue. */
4220 129036 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4221 : /* We need to peel exactly one iteration for gaps. */
4222 28 : peel_iters_epilogue = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
4223 129008 : else if (npeel < 0)
4224 : {
4225 : /* If peeling for alignment is unknown, loop bound of main loop
4226 : becomes unknown. */
4227 282 : peel_iters_epilogue = assumed_vf / 2;
4228 282 : if (dump_enabled_p ())
4229 11 : dump_printf (MSG_NOTE, "cost model: "
4230 : "epilogue peel iters set to vf/2 because "
4231 : "peeling for alignment is unknown.\n");
4232 :
4233 : /* See the same reason above in peel_iters_prologue calculation. */
4234 : epilogue_need_br_taken_cost = true;
4235 : epilogue_need_br_not_taken_cost = true;
4236 : }
4237 : else
4238 : {
4239 128726 : peel_iters_epilogue = vect_get_peel_iters_epilogue (loop_vinfo, npeel);
4240 128726 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_epilogue > 0)
4241 : /* If peeled iterations are known but number of scalar loop
4242 : iterations are unknown, count a taken branch per peeled loop. */
4243 129036 : epilogue_need_br_taken_cost = true;
4244 : }
4245 :
4246 : /* The way we cummulate peeling costs into the vector prologue/epilogue
4247 : cost is a bit awkward given we cannot reuse scalar_costs which is
4248 : already computed and also because it cannot take into account any
4249 : epilogue vectorization we'll carry out in the end. */
4250 :
4251 129036 : stmt_info_for_cost *si;
4252 129036 : int j;
4253 : /* Add costs associated with peel_iters_prologue. */
4254 129036 : if (peel_iters_prologue)
4255 1083 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4256 : {
4257 785 : int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si,
4258 : peel_iters_prologue);
4259 785 : (void) add_stmt_cost (target_cost_data, count, si->kind,
4260 : si->stmt_info, si->node, si->vectype,
4261 : si->misalign, vect_prologue);
4262 : }
4263 :
4264 : /* Add costs associated with peel_iters_epilogue. */
4265 129036 : if (peel_iters_epilogue)
4266 396834 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4267 : {
4268 317150 : int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si,
4269 : peel_iters_epilogue);
4270 317150 : (void) add_stmt_cost (target_cost_data, count, si->kind,
4271 : si->stmt_info, si->node, si->vectype,
4272 : si->misalign, vect_epilogue);
4273 : }
4274 :
4275 : /* Add possible cond_branch_taken/cond_branch_not_taken cost. */
4276 :
4277 129036 : if (prologue_need_br_taken_cost)
4278 282 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4279 : vect_prologue);
4280 :
4281 129036 : if (prologue_need_br_not_taken_cost)
4282 282 : (void) add_stmt_cost (target_cost_data, 1,
4283 : cond_branch_not_taken, vect_prologue);
4284 :
4285 129036 : if (epilogue_need_br_taken_cost)
4286 67386 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4287 : vect_epilogue);
4288 :
4289 129036 : if (epilogue_need_br_not_taken_cost)
4290 282 : (void) add_stmt_cost (target_cost_data, 1,
4291 : cond_branch_not_taken, vect_epilogue);
4292 :
4293 : /* Take care of special costs for rgroup controls of partial vectors. */
4294 28 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4295 129064 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4296 : == vect_partial_vectors_avx512))
4297 : {
4298 : /* Calculate how many masks we need to generate. */
4299 28 : unsigned int num_masks = 0;
4300 28 : bool need_saturation = false;
4301 115 : for (auto rgm : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
4302 31 : if (rgm.type)
4303 : {
4304 28 : unsigned nvectors = rgm.factor;
4305 28 : num_masks += nvectors;
4306 28 : if (TYPE_PRECISION (TREE_TYPE (rgm.compare_type))
4307 28 : < TYPE_PRECISION (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo)))
4308 10 : need_saturation = true;
4309 : }
4310 :
4311 : /* ??? The target isn't able to identify the costs below as
4312 : producing masks so it cannot penaltize cases where we'd run
4313 : out of mask registers for example. */
4314 :
4315 : /* ??? We are also failing to account for smaller vector masks
4316 : we generate by splitting larger masks in vect_get_loop_mask. */
4317 :
4318 : /* In the worst case, we need to generate each mask in the prologue
4319 : and in the loop body. We need one splat per group and one
4320 : compare per mask.
4321 :
4322 : Sometimes the prologue mask will fold to a constant,
4323 : so the actual prologue cost might be smaller. However, it's
4324 : simpler and safer to use the worst-case cost; if this ends up
4325 : being the tie-breaker between vectorizing or not, then it's
4326 : probably better not to vectorize. */
4327 28 : (void) add_stmt_cost (target_cost_data,
4328 : num_masks
4329 28 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4330 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4331 : vect_prologue);
4332 56 : (void) add_stmt_cost (target_cost_data,
4333 : num_masks
4334 56 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4335 : vector_stmt, NULL, NULL, NULL_TREE, 0, vect_body);
4336 :
4337 : /* When we need saturation we need it both in the prologue and
4338 : the epilogue. */
4339 28 : if (need_saturation)
4340 : {
4341 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4342 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4343 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4344 : NULL, NULL, NULL_TREE, 0, vect_body);
4345 : }
4346 : }
4347 0 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4348 129008 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4349 : == vect_partial_vectors_while_ult))
4350 : {
4351 : /* Calculate how many masks we need to generate. */
4352 : unsigned int num_masks = 0;
4353 : rgroup_controls *rgm;
4354 : unsigned int num_vectors_m1;
4355 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec,
4356 : num_vectors_m1, rgm)
4357 0 : if (rgm->type)
4358 0 : num_masks += num_vectors_m1 + 1;
4359 0 : gcc_assert (num_masks > 0);
4360 :
4361 : /* In the worst case, we need to generate each mask in the prologue
4362 : and in the loop body. One of the loop body mask instructions
4363 : replaces the comparison in the scalar loop, and since we don't
4364 : count the scalar comparison against the scalar body, we shouldn't
4365 : count that vector instruction against the vector body either.
4366 :
4367 : Sometimes we can use unpacks instead of generating prologue
4368 : masks and sometimes the prologue mask will fold to a constant,
4369 : so the actual prologue cost might be smaller. However, it's
4370 : simpler and safer to use the worst-case cost; if this ends up
4371 : being the tie-breaker between vectorizing or not, then it's
4372 : probably better not to vectorize. */
4373 0 : (void) add_stmt_cost (target_cost_data, num_masks,
4374 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4375 : vect_prologue);
4376 0 : (void) add_stmt_cost (target_cost_data, num_masks - 1,
4377 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4378 : vect_body);
4379 : }
4380 129008 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
4381 : {
4382 : /* Referring to the functions vect_set_loop_condition_partial_vectors
4383 : and vect_set_loop_controls_directly, we need to generate each
4384 : length in the prologue and in the loop body if required. Although
4385 : there are some possible optimizations, we consider the worst case
4386 : here. */
4387 :
4388 0 : bool niters_known_p = LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo);
4389 0 : signed char partial_load_store_bias
4390 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
4391 0 : bool need_iterate_p
4392 0 : = (!LOOP_VINFO_EPILOGUE_P (loop_vinfo)
4393 0 : && !vect_known_niters_smaller_than_vf (loop_vinfo));
4394 :
4395 : /* Calculate how many statements to be added. */
4396 0 : unsigned int prologue_stmts = 0;
4397 0 : unsigned int body_stmts = 0;
4398 :
4399 0 : rgroup_controls *rgc;
4400 0 : unsigned int num_vectors_m1;
4401 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), num_vectors_m1, rgc)
4402 0 : if (rgc->type)
4403 : {
4404 : /* May need one SHIFT for nitems_total computation. */
4405 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
4406 0 : if (nitems != 1 && !niters_known_p)
4407 0 : prologue_stmts += 1;
4408 :
4409 : /* May need one MAX and one MINUS for wrap around. */
4410 0 : if (vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc))
4411 0 : prologue_stmts += 2;
4412 :
4413 : /* Need one MAX and one MINUS for each batch limit excepting for
4414 : the 1st one. */
4415 0 : prologue_stmts += num_vectors_m1 * 2;
4416 :
4417 0 : unsigned int num_vectors = num_vectors_m1 + 1;
4418 :
4419 : /* Need to set up lengths in prologue, only one MIN required
4420 : for each since start index is zero. */
4421 0 : prologue_stmts += num_vectors;
4422 :
4423 : /* If we have a non-zero partial load bias, we need one PLUS
4424 : to adjust the load length. */
4425 0 : if (partial_load_store_bias != 0)
4426 0 : body_stmts += 1;
4427 :
4428 0 : unsigned int length_update_cost = 0;
4429 0 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
4430 : /* For decrement IV style, Each only need a single SELECT_VL
4431 : or MIN since beginning to calculate the number of elements
4432 : need to be processed in current iteration. */
4433 : length_update_cost = 1;
4434 : else
4435 : /* For increment IV stype, Each may need two MINs and one MINUS to
4436 : update lengths in body for next iteration. */
4437 0 : length_update_cost = 3;
4438 :
4439 0 : if (need_iterate_p)
4440 0 : body_stmts += length_update_cost * num_vectors;
4441 : }
4442 :
4443 0 : (void) add_stmt_cost (target_cost_data, prologue_stmts,
4444 : scalar_stmt, vect_prologue);
4445 0 : (void) add_stmt_cost (target_cost_data, body_stmts,
4446 : scalar_stmt, vect_body);
4447 : }
4448 :
4449 : /* FORNOW: The scalar outside cost is incremented in one of the
4450 : following ways:
4451 :
4452 : 1. The vectorizer checks for alignment and aliasing and generates
4453 : a condition that allows dynamic vectorization. A cost model
4454 : check is ANDED with the versioning condition. Hence scalar code
4455 : path now has the added cost of the versioning check.
4456 :
4457 : if (cost > th & versioning_check)
4458 : jmp to vector code
4459 :
4460 : Hence run-time scalar is incremented by not-taken branch cost.
4461 :
4462 : 2. The vectorizer then checks if a prologue is required. If the
4463 : cost model check was not done before during versioning, it has to
4464 : be done before the prologue check.
4465 :
4466 : if (cost <= th)
4467 : prologue = scalar_iters
4468 : if (prologue == 0)
4469 : jmp to vector code
4470 : else
4471 : execute prologue
4472 : if (prologue == num_iters)
4473 : go to exit
4474 :
4475 : Hence the run-time scalar cost is incremented by a taken branch,
4476 : plus a not-taken branch, plus a taken branch cost.
4477 :
4478 : 3. The vectorizer then checks if an epilogue is required. If the
4479 : cost model check was not done before during prologue check, it
4480 : has to be done with the epilogue check.
4481 :
4482 : if (prologue == 0)
4483 : jmp to vector code
4484 : else
4485 : execute prologue
4486 : if (prologue == num_iters)
4487 : go to exit
4488 : vector code:
4489 : if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
4490 : jmp to epilogue
4491 :
4492 : Hence the run-time scalar cost should be incremented by 2 taken
4493 : branches.
4494 :
4495 : TODO: The back end may reorder the BBS's differently and reverse
4496 : conditions/branch directions. Change the estimates below to
4497 : something more reasonable. */
4498 :
4499 : /* If the number of iterations is known and we do not do versioning, we can
4500 : decide whether to vectorize at compile time. Hence the scalar version
4501 : do not carry cost model guard costs. */
4502 60763 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
4503 189799 : || LOOP_REQUIRES_VERSIONING (loop_vinfo))
4504 : {
4505 : /* Cost model check occurs at versioning. */
4506 69406 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4507 8141 : scalar_outside_cost += vect_get_stmt_cost (cond_branch_not_taken);
4508 : else
4509 : {
4510 : /* Cost model check occurs at prologue generation. */
4511 61265 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
4512 155 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken)
4513 155 : + vect_get_stmt_cost (cond_branch_not_taken);
4514 : /* Cost model check occurs at epilogue generation. */
4515 : else
4516 61110 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken);
4517 : }
4518 : }
4519 :
4520 : /* Complete the target-specific cost calculations. */
4521 129036 : loop_vinfo->vector_costs->finish_cost (loop_vinfo->scalar_costs);
4522 129036 : vec_prologue_cost = loop_vinfo->vector_costs->prologue_cost ();
4523 129036 : vec_inside_cost = loop_vinfo->vector_costs->body_cost ();
4524 129036 : vec_epilogue_cost = loop_vinfo->vector_costs->epilogue_cost ();
4525 129036 : if (suggested_unroll_factor)
4526 128646 : *suggested_unroll_factor
4527 128646 : = loop_vinfo->vector_costs->suggested_unroll_factor ();
4528 :
4529 128646 : if (suggested_unroll_factor && *suggested_unroll_factor > 1
4530 419 : && LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) != MAX_VECTORIZATION_FACTOR
4531 0 : && !known_le (LOOP_VINFO_VECT_FACTOR (loop_vinfo) *
4532 : *suggested_unroll_factor,
4533 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo)))
4534 : {
4535 0 : if (dump_enabled_p ())
4536 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4537 : "can't unroll as unrolled vectorization factor larger"
4538 : " than maximum vectorization factor: "
4539 : HOST_WIDE_INT_PRINT_UNSIGNED "\n",
4540 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo));
4541 0 : *suggested_unroll_factor = 1;
4542 : }
4543 :
4544 129036 : vec_outside_cost = (int)(vec_prologue_cost + vec_epilogue_cost);
4545 :
4546 129036 : if (dump_enabled_p ())
4547 : {
4548 1159 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
4549 1159 : dump_printf (MSG_NOTE, " Vector inside of loop cost: %d\n",
4550 : vec_inside_cost);
4551 1159 : dump_printf (MSG_NOTE, " Vector prologue cost: %d\n",
4552 : vec_prologue_cost);
4553 1159 : dump_printf (MSG_NOTE, " Vector epilogue cost: %d\n",
4554 : vec_epilogue_cost);
4555 1159 : dump_printf (MSG_NOTE, " Scalar iteration cost: %d\n",
4556 : scalar_single_iter_cost);
4557 1159 : dump_printf (MSG_NOTE, " Scalar outside cost: %d\n",
4558 : scalar_outside_cost);
4559 1159 : dump_printf (MSG_NOTE, " Vector outside cost: %d\n",
4560 : vec_outside_cost);
4561 1159 : dump_printf (MSG_NOTE, " prologue iterations: %d\n",
4562 : peel_iters_prologue);
4563 1159 : dump_printf (MSG_NOTE, " epilogue iterations: %d\n",
4564 : peel_iters_epilogue);
4565 : }
4566 :
4567 : /* Calculate number of iterations required to make the vector version
4568 : profitable, relative to the loop bodies only. The following condition
4569 : must hold true:
4570 : SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
4571 : where
4572 : SIC = scalar iteration cost, VIC = vector iteration cost,
4573 : VOC = vector outside cost, VF = vectorization factor,
4574 : NPEEL = prologue iterations + epilogue iterations,
4575 : SOC = scalar outside cost for run time cost model check. */
4576 :
4577 129036 : int saving_per_viter = (scalar_single_iter_cost * assumed_vf
4578 129036 : - vec_inside_cost);
4579 129036 : if (saving_per_viter <= 0)
4580 : {
4581 24962 : if (LOOP_VINFO_LOOP (loop_vinfo)->force_vectorize)
4582 0 : warning_at (vect_location.get_location_t (), OPT_Wopenmp_simd,
4583 : "vectorization did not happen for a simd loop");
4584 :
4585 24962 : if (dump_enabled_p ())
4586 38 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4587 : "cost model: the vector iteration cost = %d "
4588 : "divided by the scalar iteration cost = %d "
4589 : "is greater or equal to the vectorization factor = %d"
4590 : ".\n",
4591 : vec_inside_cost, scalar_single_iter_cost, assumed_vf);
4592 24962 : *ret_min_profitable_niters = -1;
4593 24962 : *ret_min_profitable_estimate = -1;
4594 24962 : return;
4595 : }
4596 :
4597 : /* ??? The "if" arm is written to handle all cases; see below for what
4598 : we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4599 104074 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4600 : {
4601 : /* Rewriting the condition above in terms of the number of
4602 : vector iterations (vniters) rather than the number of
4603 : scalar iterations (niters) gives:
4604 :
4605 : SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
4606 :
4607 : <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
4608 :
4609 : For integer N, X and Y when X > 0:
4610 :
4611 : N * X > Y <==> N >= (Y /[floor] X) + 1. */
4612 20 : int outside_overhead = (vec_outside_cost
4613 20 : - scalar_single_iter_cost * peel_iters_prologue
4614 20 : - scalar_single_iter_cost * peel_iters_epilogue
4615 : - scalar_outside_cost);
4616 : /* We're only interested in cases that require at least one
4617 : vector iteration. */
4618 20 : int min_vec_niters = 1;
4619 20 : if (outside_overhead > 0)
4620 16 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4621 :
4622 20 : if (dump_enabled_p ())
4623 8 : dump_printf (MSG_NOTE, " Minimum number of vector iterations: %d\n",
4624 : min_vec_niters);
4625 :
4626 20 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4627 : {
4628 : /* Now that we know the minimum number of vector iterations,
4629 : find the minimum niters for which the scalar cost is larger:
4630 :
4631 : SIC * niters > VIC * vniters + VOC - SOC
4632 :
4633 : We know that the minimum niters is no more than
4634 : vniters * VF + NPEEL, but it might be (and often is) less
4635 : than that if a partial vector iteration is cheaper than the
4636 : equivalent scalar code. */
4637 20 : int threshold = (vec_inside_cost * min_vec_niters
4638 20 : + vec_outside_cost
4639 20 : - scalar_outside_cost);
4640 20 : if (threshold <= 0)
4641 : min_profitable_iters = 1;
4642 : else
4643 20 : min_profitable_iters = threshold / scalar_single_iter_cost + 1;
4644 : }
4645 : else
4646 : /* Convert the number of vector iterations into a number of
4647 : scalar iterations. */
4648 0 : min_profitable_iters = (min_vec_niters * assumed_vf
4649 0 : + peel_iters_prologue
4650 : + peel_iters_epilogue);
4651 : }
4652 : else
4653 : {
4654 104054 : min_profitable_iters = ((vec_outside_cost - scalar_outside_cost)
4655 104054 : * assumed_vf
4656 104054 : - vec_inside_cost * peel_iters_prologue
4657 104054 : - vec_inside_cost * peel_iters_epilogue);
4658 104054 : if (min_profitable_iters <= 0)
4659 : min_profitable_iters = 0;
4660 : else
4661 : {
4662 87873 : min_profitable_iters /= saving_per_viter;
4663 :
4664 87873 : if ((scalar_single_iter_cost * assumed_vf * min_profitable_iters)
4665 87873 : <= (((int) vec_inside_cost * min_profitable_iters)
4666 87873 : + (((int) vec_outside_cost - scalar_outside_cost)
4667 : * assumed_vf)))
4668 87873 : min_profitable_iters++;
4669 : }
4670 : }
4671 :
4672 104074 : if (dump_enabled_p ())
4673 1121 : dump_printf (MSG_NOTE,
4674 : " Calculated minimum iters for profitability: %d\n",
4675 : min_profitable_iters);
4676 :
4677 104074 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
4678 104054 : && min_profitable_iters < (assumed_vf + peel_iters_prologue))
4679 : /* We want the vectorized loop to execute at least once. */
4680 : min_profitable_iters = assumed_vf + peel_iters_prologue;
4681 23765 : else if (min_profitable_iters < peel_iters_prologue)
4682 : /* For LOOP_VINFO_USING_PARTIAL_VECTORS_P, we need to ensure the
4683 : vectorized loop executes at least once. */
4684 : min_profitable_iters = peel_iters_prologue;
4685 :
4686 104074 : if (dump_enabled_p ())
4687 1121 : dump_printf_loc (MSG_NOTE, vect_location,
4688 : " Runtime profitability threshold = %d\n",
4689 : min_profitable_iters);
4690 :
4691 104074 : *ret_min_profitable_niters = min_profitable_iters;
4692 :
4693 : /* Calculate number of iterations required to make the vector version
4694 : profitable, relative to the loop bodies only.
4695 :
4696 : Non-vectorized variant is SIC * niters and it must win over vector
4697 : variant on the expected loop trip count. The following condition must hold true:
4698 : SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC */
4699 :
4700 104074 : if (vec_outside_cost <= 0)
4701 : min_profitable_estimate = 0;
4702 : /* ??? This "else if" arm is written to handle all cases; see below for
4703 : what we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4704 92702 : else if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4705 : {
4706 : /* This is a repeat of the code above, but with + SOC rather
4707 : than - SOC. */
4708 20 : int outside_overhead = (vec_outside_cost
4709 20 : - scalar_single_iter_cost * peel_iters_prologue
4710 20 : - scalar_single_iter_cost * peel_iters_epilogue
4711 : + scalar_outside_cost);
4712 20 : int min_vec_niters = 1;
4713 20 : if (outside_overhead > 0)
4714 20 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4715 :
4716 20 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4717 : {
4718 20 : int threshold = (vec_inside_cost * min_vec_niters
4719 20 : + vec_outside_cost
4720 20 : + scalar_outside_cost);
4721 20 : min_profitable_estimate = threshold / scalar_single_iter_cost + 1;
4722 : }
4723 : else
4724 : min_profitable_estimate = (min_vec_niters * assumed_vf
4725 : + peel_iters_prologue
4726 : + peel_iters_epilogue);
4727 : }
4728 : else
4729 : {
4730 92682 : min_profitable_estimate = ((vec_outside_cost + scalar_outside_cost)
4731 92682 : * assumed_vf
4732 92682 : - vec_inside_cost * peel_iters_prologue
4733 92682 : - vec_inside_cost * peel_iters_epilogue)
4734 92682 : / ((scalar_single_iter_cost * assumed_vf)
4735 : - vec_inside_cost);
4736 : }
4737 104074 : min_profitable_estimate = MAX (min_profitable_estimate, min_profitable_iters);
4738 104074 : if (dump_enabled_p ())
4739 1121 : dump_printf_loc (MSG_NOTE, vect_location,
4740 : " Static estimate profitability threshold = %d\n",
4741 : min_profitable_estimate);
4742 :
4743 104074 : *ret_min_profitable_estimate = min_profitable_estimate;
4744 : }
4745 :
4746 : /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
4747 : vector elements (not bits) for a vector with NELT elements. */
4748 : static void
4749 2104 : calc_vec_perm_mask_for_shift (unsigned int offset, unsigned int nelt,
4750 : vec_perm_builder *sel)
4751 : {
4752 : /* The encoding is a single stepped pattern. Any wrap-around is handled
4753 : by vec_perm_indices. */
4754 2104 : sel->new_vector (nelt, 1, 3);
4755 8416 : for (unsigned int i = 0; i < 3; i++)
4756 6312 : sel->quick_push (i + offset);
4757 2104 : }
4758 :
4759 : /* Checks whether the target supports whole-vector shifts for vectors of mode
4760 : MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
4761 : it supports vec_perm_const with masks for all necessary shift amounts. */
4762 : static bool
4763 13398 : have_whole_vector_shift (machine_mode mode)
4764 : {
4765 13398 : if (can_implement_p (vec_shr_optab, mode))
4766 : return true;
4767 :
4768 : /* Variable-length vectors should be handled via the optab. */
4769 63 : unsigned int nelt;
4770 126 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
4771 : return false;
4772 :
4773 63 : vec_perm_builder sel;
4774 63 : vec_perm_indices indices;
4775 315 : for (unsigned int i = nelt / 2; i >= 1; i /= 2)
4776 : {
4777 252 : calc_vec_perm_mask_for_shift (i, nelt, &sel);
4778 252 : indices.new_vector (sel, 2, nelt);
4779 252 : if (!can_vec_perm_const_p (mode, mode, indices, false))
4780 : return false;
4781 : }
4782 : return true;
4783 63 : }
4784 :
4785 : /* Return true if (a) STMT_INFO is a DOT_PROD_EXPR reduction whose
4786 : multiplication operands have differing signs and (b) we intend
4787 : to emulate the operation using a series of signed DOT_PROD_EXPRs.
4788 : See vect_emulate_mixed_dot_prod for the actual sequence used. */
4789 :
4790 : static bool
4791 2455 : vect_is_emulated_mixed_dot_prod (slp_tree slp_node)
4792 : {
4793 2455 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_node);
4794 2455 : gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
4795 2000 : if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
4796 : return false;
4797 :
4798 856 : tree rhs1 = gimple_assign_rhs1 (assign);
4799 856 : tree rhs2 = gimple_assign_rhs2 (assign);
4800 856 : if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
4801 : return false;
4802 :
4803 645 : return !directly_supported_p (DOT_PROD_EXPR,
4804 : SLP_TREE_VECTYPE (slp_node),
4805 215 : SLP_TREE_VECTYPE
4806 : (SLP_TREE_CHILDREN (slp_node)[0]),
4807 215 : optab_vector_mixed_sign);
4808 : }
4809 :
4810 : /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
4811 : functions. Design better to avoid maintenance issues. */
4812 :
4813 : /* Function vect_model_reduction_cost.
4814 :
4815 : Models cost for a reduction operation, including the vector ops
4816 : generated within the strip-mine loop in some cases, the initial
4817 : definition before the loop, and the epilogue code that must be generated. */
4818 :
4819 : static void
4820 70007 : vect_model_reduction_cost (loop_vec_info loop_vinfo,
4821 : slp_tree node, internal_fn reduc_fn,
4822 : vect_reduction_type reduction_type,
4823 : int ncopies, stmt_vector_for_cost *cost_vec)
4824 : {
4825 70007 : int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
4826 70007 : tree vectype;
4827 70007 : machine_mode mode;
4828 70007 : class loop *loop = NULL;
4829 :
4830 70007 : if (loop_vinfo)
4831 70007 : loop = LOOP_VINFO_LOOP (loop_vinfo);
4832 :
4833 : /* Condition reductions generate two reductions in the loop. */
4834 70007 : if (reduction_type == COND_REDUCTION)
4835 349 : ncopies *= 2;
4836 :
4837 70007 : vectype = SLP_TREE_VECTYPE (node);
4838 70007 : mode = TYPE_MODE (vectype);
4839 70007 : stmt_vec_info orig_stmt_info
4840 70007 : = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
4841 :
4842 70007 : gimple_match_op op;
4843 70007 : if (!gimple_extract_op (orig_stmt_info->stmt, &op))
4844 0 : gcc_unreachable ();
4845 :
4846 70007 : if (reduction_type == EXTRACT_LAST_REDUCTION)
4847 : /* No extra instructions are needed in the prologue. The loop body
4848 : operations are costed in vectorizable_condition. */
4849 : inside_cost = 0;
4850 70007 : else if (reduction_type == FOLD_LEFT_REDUCTION)
4851 : {
4852 : /* No extra instructions needed in the prologue. */
4853 4714 : prologue_cost = 0;
4854 :
4855 4714 : if (reduc_fn != IFN_LAST)
4856 : /* Count one reduction-like operation per vector. */
4857 0 : inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
4858 : node, 0, vect_body);
4859 : else
4860 : {
4861 : /* Use NCOPIES deconstructs and NELEMENTS scalar ops. */
4862 4714 : unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
4863 4714 : inside_cost = record_stmt_cost (cost_vec, ncopies,
4864 : vec_deconstruct, node, 0,
4865 : vect_body);
4866 4714 : inside_cost += record_stmt_cost (cost_vec, nelements,
4867 : scalar_stmt, node, 0,
4868 : vect_body);
4869 : }
4870 : }
4871 : else
4872 : {
4873 : /* Add in the cost of the initial definitions. */
4874 65293 : int prologue_stmts;
4875 65293 : if (reduction_type == COND_REDUCTION)
4876 : /* For cond reductions we have four vectors: initial index, step,
4877 : initial result of the data reduction, initial value of the index
4878 : reduction. */
4879 : prologue_stmts = 4;
4880 : else
4881 : /* We need the initial reduction value. */
4882 64944 : prologue_stmts = 1;
4883 65293 : prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
4884 : scalar_to_vec, node, 0,
4885 : vect_prologue);
4886 : }
4887 :
4888 : /* Determine cost of epilogue code.
4889 :
4890 : We have a reduction operator that will reduce the vector in one statement.
4891 : Also requires scalar extract. */
4892 :
4893 70007 : if (!loop || !nested_in_vect_loop_p (loop, orig_stmt_info))
4894 : {
4895 69831 : if (reduc_fn != IFN_LAST)
4896 : {
4897 49927 : if (reduction_type == COND_REDUCTION)
4898 : {
4899 : /* An EQ stmt and an COND_EXPR stmt. */
4900 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4901 : vector_stmt, node, 0,
4902 : vect_epilogue);
4903 : /* Reduction of the max index and a reduction of the found
4904 : values. */
4905 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4906 : vec_to_scalar, node, 0,
4907 : vect_epilogue);
4908 : /* A broadcast of the max value. */
4909 18 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4910 : scalar_to_vec, node, 0,
4911 : vect_epilogue);
4912 : }
4913 : else
4914 : {
4915 49909 : epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
4916 : node, 0, vect_epilogue);
4917 49909 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4918 : vec_to_scalar, node, 0,
4919 : vect_epilogue);
4920 : }
4921 : }
4922 19904 : else if (reduction_type == COND_REDUCTION)
4923 : {
4924 331 : unsigned estimated_nunits = vect_nunits_for_cost (vectype);
4925 : /* Extraction of scalar elements. */
4926 331 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4927 : vec_deconstruct, node, 0,
4928 : vect_epilogue);
4929 : /* Scalar max reductions via COND_EXPR / MAX_EXPR. */
4930 331 : epilogue_cost += record_stmt_cost (cost_vec,
4931 331 : 2 * estimated_nunits - 3,
4932 : scalar_stmt, node, 0,
4933 : vect_epilogue);
4934 : }
4935 19573 : else if (reduction_type == EXTRACT_LAST_REDUCTION
4936 19573 : || reduction_type == FOLD_LEFT_REDUCTION)
4937 : /* No extra instructions need in the epilogue. */
4938 : ;
4939 : else
4940 : {
4941 14859 : int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
4942 14859 : tree bitsize = TYPE_SIZE (op.type);
4943 14859 : int element_bitsize = tree_to_uhwi (bitsize);
4944 14859 : int nelements = vec_size_in_bits / element_bitsize;
4945 :
4946 14859 : if (op.code == COND_EXPR)
4947 31 : op.code = MAX_EXPR;
4948 :
4949 : /* We have a whole vector shift available. */
4950 2519 : if (VECTOR_MODE_P (mode)
4951 14859 : && directly_supported_p (op.code, vectype)
4952 26440 : && have_whole_vector_shift (mode))
4953 : {
4954 : /* Final reduction via vector shifts and the reduction operator.
4955 : Also requires scalar extract. */
4956 34743 : epilogue_cost += record_stmt_cost (cost_vec,
4957 23162 : exact_log2 (nelements) * 2,
4958 : vector_stmt, node, 0,
4959 : vect_epilogue);
4960 11581 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4961 : vec_to_scalar, node, 0,
4962 : vect_epilogue);
4963 : }
4964 : else
4965 : /* Use extracts and reduction op for final reduction. For N
4966 : elements, we have N extracts and N-1 reduction ops. */
4967 3278 : epilogue_cost += record_stmt_cost (cost_vec,
4968 3278 : nelements + nelements - 1,
4969 : vector_stmt, node, 0,
4970 : vect_epilogue);
4971 : }
4972 : }
4973 :
4974 70007 : if (dump_enabled_p ())
4975 3100 : dump_printf (MSG_NOTE,
4976 : "vect_model_reduction_cost: inside_cost = %d, "
4977 : "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost,
4978 : prologue_cost, epilogue_cost);
4979 70007 : }
4980 :
4981 : /* SEQ is a sequence of instructions that initialize the reduction
4982 : described by REDUC_INFO. Emit them in the appropriate place. */
4983 :
4984 : static void
4985 461 : vect_emit_reduction_init_stmts (loop_vec_info loop_vinfo,
4986 : vect_reduc_info reduc_info, gimple *seq)
4987 : {
4988 461 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
4989 : {
4990 : /* When reusing an accumulator from the main loop, we only need
4991 : initialization instructions if the main loop can be skipped.
4992 : In that case, emit the initialization instructions at the end
4993 : of the guard block that does the skip. */
4994 18 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
4995 18 : gcc_assert (skip_edge);
4996 18 : gimple_stmt_iterator gsi = gsi_last_bb (skip_edge->src);
4997 18 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
4998 : }
4999 : else
5000 : {
5001 : /* The normal case: emit the initialization instructions on the
5002 : preheader edge. */
5003 443 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
5004 443 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), seq);
5005 : }
5006 461 : }
5007 :
5008 : /* Get at the initial defs for the reduction PHIs for REDUC_INFO,
5009 : which performs a reduction involving GROUP_SIZE scalar statements.
5010 : NUMBER_OF_VECTORS is the number of vector defs to create. If NEUTRAL_OP
5011 : is nonnull, introducing extra elements of that value will not change the
5012 : result. */
5013 :
5014 : static void
5015 21821 : get_initial_defs_for_reduction (loop_vec_info loop_vinfo,
5016 : vect_reduc_info reduc_info,
5017 : tree vector_type,
5018 : vec<tree> *vec_oprnds,
5019 : unsigned int number_of_vectors,
5020 : unsigned int group_size, tree neutral_op)
5021 : {
5022 21821 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
5023 21821 : unsigned HOST_WIDE_INT nunits;
5024 21821 : unsigned j, number_of_places_left_in_vector;
5025 21821 : unsigned int i;
5026 :
5027 43642 : gcc_assert (group_size == initial_values.length () || neutral_op);
5028 :
5029 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
5030 : created vectors. It is greater than 1 if unrolling is performed.
5031 :
5032 : For example, we have two scalar operands, s1 and s2 (e.g., group of
5033 : strided accesses of size two), while NUNITS is four (i.e., four scalars
5034 : of this type can be packed in a vector). The output vector will contain
5035 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
5036 : will be 2).
5037 :
5038 : If GROUP_SIZE > NUNITS, the scalars will be split into several
5039 : vectors containing the operands.
5040 :
5041 : For example, NUNITS is four as before, and the group size is 8
5042 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
5043 : {s5, s6, s7, s8}. */
5044 :
5045 21821 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
5046 : nunits = group_size;
5047 :
5048 21821 : tree vector_elt_type = TREE_TYPE (vector_type);
5049 21821 : number_of_places_left_in_vector = nunits;
5050 21821 : bool constant_p = true;
5051 21821 : tree_vector_builder elts (vector_type, nunits, 1);
5052 21821 : elts.quick_grow (nunits);
5053 21821 : gimple_seq ctor_seq = NULL;
5054 21821 : if (neutral_op
5055 43048 : && !useless_type_conversion_p (vector_elt_type,
5056 21227 : TREE_TYPE (neutral_op)))
5057 : {
5058 248 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
5059 227 : neutral_op = gimple_build (&ctor_seq, COND_EXPR,
5060 : vector_elt_type,
5061 : neutral_op,
5062 : build_all_ones_cst (vector_elt_type),
5063 : build_zero_cst (vector_elt_type));
5064 : else
5065 21 : neutral_op = gimple_convert (&ctor_seq, vector_elt_type, neutral_op);
5066 : }
5067 204605 : for (j = 0; j < nunits * number_of_vectors; ++j)
5068 : {
5069 182784 : tree op;
5070 182784 : i = j % group_size;
5071 :
5072 : /* Get the def before the loop. In reduction chain we have only
5073 : one initial value. Else we have as many as PHIs in the group. */
5074 182784 : if (i >= initial_values.length () || (j > i && neutral_op))
5075 : op = neutral_op;
5076 : else
5077 : {
5078 51550 : if (!useless_type_conversion_p (vector_elt_type,
5079 25775 : TREE_TYPE (initial_values[i])))
5080 : {
5081 263 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
5082 478 : initial_values[i] = gimple_build (&ctor_seq, COND_EXPR,
5083 : vector_elt_type,
5084 239 : initial_values[i],
5085 : build_all_ones_cst
5086 : (vector_elt_type),
5087 : build_zero_cst
5088 : (vector_elt_type));
5089 : else
5090 48 : initial_values[i] = gimple_convert (&ctor_seq,
5091 : vector_elt_type,
5092 24 : initial_values[i]);
5093 : }
5094 25775 : op = initial_values[i];
5095 : }
5096 :
5097 : /* Create 'vect_ = {op0,op1,...,opn}'. */
5098 182784 : number_of_places_left_in_vector--;
5099 182784 : elts[nunits - number_of_places_left_in_vector - 1] = op;
5100 182784 : if (!CONSTANT_CLASS_P (op))
5101 2499 : constant_p = false;
5102 :
5103 182784 : if (number_of_places_left_in_vector == 0)
5104 : {
5105 23405 : tree init;
5106 46810 : if (constant_p && !neutral_op
5107 46516 : ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type), nunits)
5108 23405 : : known_eq (TYPE_VECTOR_SUBPARTS (vector_type), nunits))
5109 : /* Build the vector directly from ELTS. */
5110 23405 : init = gimple_build_vector (&ctor_seq, &elts);
5111 0 : else if (neutral_op)
5112 : {
5113 : /* Build a vector of the neutral value and shift the
5114 : other elements into place. */
5115 0 : init = gimple_build_vector_from_val (&ctor_seq, vector_type,
5116 : neutral_op);
5117 0 : int k = nunits;
5118 0 : while (k > 0 && operand_equal_p (elts[k - 1], neutral_op))
5119 0 : k -= 1;
5120 0 : while (k > 0)
5121 : {
5122 0 : k -= 1;
5123 0 : init = gimple_build (&ctor_seq, CFN_VEC_SHL_INSERT,
5124 0 : vector_type, init, elts[k]);
5125 : }
5126 : }
5127 : else
5128 : {
5129 : /* First time round, duplicate ELTS to fill the
5130 : required number of vectors. */
5131 0 : duplicate_and_interleave (loop_vinfo, &ctor_seq, vector_type,
5132 : elts, number_of_vectors, *vec_oprnds);
5133 0 : break;
5134 : }
5135 23405 : vec_oprnds->quick_push (init);
5136 :
5137 23405 : number_of_places_left_in_vector = nunits;
5138 23405 : elts.new_vector (vector_type, nunits, 1);
5139 23405 : elts.quick_grow (nunits);
5140 23405 : constant_p = true;
5141 : }
5142 : }
5143 21821 : if (ctor_seq != NULL)
5144 461 : vect_emit_reduction_init_stmts (loop_vinfo, reduc_info, ctor_seq);
5145 21821 : }
5146 :
5147 : vect_reduc_info
5148 213477 : info_for_reduction (loop_vec_info loop_vinfo, slp_tree node)
5149 : {
5150 213477 : gcc_assert (node);
5151 213477 : if (node->cycle_info.id == -1)
5152 : return NULL;
5153 208677 : return loop_vinfo->reduc_infos[node->cycle_info.id];
5154 : }
5155 :
5156 : /* See if LOOP_VINFO is an epilogue loop whose main loop had a reduction that
5157 : REDUC_INFO can build on. Adjust REDUC_INFO and return true if so, otherwise
5158 : return false. */
5159 :
5160 : static bool
5161 21379 : vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
5162 : vect_reduc_info reduc_info, tree vectype)
5163 : {
5164 21379 : loop_vec_info main_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
5165 21379 : if (!main_loop_vinfo)
5166 : return false;
5167 :
5168 4497 : if (VECT_REDUC_INFO_TYPE (reduc_info) != TREE_CODE_REDUCTION)
5169 : return false;
5170 :
5171 : /* We are not set up to handle vector bools when they are not mapped
5172 : to vector integer data types. */
5173 4479 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5174 4531 : && GET_MODE_CLASS (TYPE_MODE (vectype)) != MODE_VECTOR_INT)
5175 : return false;
5176 :
5177 4477 : unsigned int num_phis = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).length ();
5178 4477 : auto_vec<tree, 16> main_loop_results (num_phis);
5179 4477 : auto_vec<tree, 16> initial_values (num_phis);
5180 4477 : if (edge main_loop_edge = loop_vinfo->main_loop_edge)
5181 : {
5182 : /* The epilogue loop can be entered either from the main loop or
5183 : from an earlier guard block. */
5184 4328 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
5185 17332 : for (tree incoming_value : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info))
5186 : {
5187 : /* Look for:
5188 :
5189 : INCOMING_VALUE = phi<MAIN_LOOP_RESULT(main loop),
5190 : INITIAL_VALUE(guard block)>. */
5191 4348 : gcc_assert (TREE_CODE (incoming_value) == SSA_NAME);
5192 :
5193 4348 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (incoming_value));
5194 4348 : gcc_assert (gimple_bb (phi) == main_loop_edge->dest);
5195 :
5196 4348 : tree from_main_loop = PHI_ARG_DEF_FROM_EDGE (phi, main_loop_edge);
5197 4348 : tree from_skip = PHI_ARG_DEF_FROM_EDGE (phi, skip_edge);
5198 :
5199 4348 : main_loop_results.quick_push (from_main_loop);
5200 4348 : initial_values.quick_push (from_skip);
5201 : }
5202 : }
5203 : else
5204 : /* The main loop dominates the epilogue loop. */
5205 149 : main_loop_results.splice (VECT_REDUC_INFO_INITIAL_VALUES (reduc_info));
5206 :
5207 : /* See if the main loop has the kind of accumulator we need. */
5208 4477 : vect_reusable_accumulator *accumulator
5209 4477 : = main_loop_vinfo->reusable_accumulators.get (main_loop_results[0]);
5210 4477 : if (!accumulator
5211 8942 : || num_phis != VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).length ()
5212 13419 : || !std::equal (main_loop_results.begin (), main_loop_results.end (),
5213 : VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).begin ()))
5214 : return false;
5215 :
5216 : /* Handle the case where we can reduce wider vectors to narrower ones. */
5217 4471 : tree old_vectype = TREE_TYPE (accumulator->reduc_input);
5218 4471 : unsigned HOST_WIDE_INT m;
5219 4471 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (old_vectype),
5220 4471 : TYPE_VECTOR_SUBPARTS (vectype), &m))
5221 0 : return false;
5222 : /* Check the intermediate vector types and operations are available. */
5223 4471 : tree prev_vectype = old_vectype;
5224 4471 : poly_uint64 intermediate_nunits = TYPE_VECTOR_SUBPARTS (old_vectype);
5225 13039 : while (known_gt (intermediate_nunits, TYPE_VECTOR_SUBPARTS (vectype)))
5226 : {
5227 4621 : intermediate_nunits = exact_div (intermediate_nunits, 2);
5228 4621 : tree intermediate_vectype = get_related_vectype_for_scalar_type
5229 4621 : (TYPE_MODE (vectype), TREE_TYPE (vectype), intermediate_nunits);
5230 9242 : if (!intermediate_vectype
5231 4621 : || !directly_supported_p (VECT_REDUC_INFO_CODE (reduc_info),
5232 : intermediate_vectype)
5233 8718 : || !can_vec_extract (TYPE_MODE (prev_vectype),
5234 8202 : TYPE_MODE (intermediate_vectype)).exists ()
5235 : /* We could actually extract VLA vectors but bit_field_ref and
5236 : friends aren't yet prepared for it. */
5237 8718 : || !TYPE_VECTOR_SUBPARTS (intermediate_vectype).is_constant ())
5238 : return false;
5239 : prev_vectype = intermediate_vectype;
5240 : }
5241 :
5242 : /* Non-SLP reductions might apply an adjustment after the reduction
5243 : operation, in order to simplify the initialization of the accumulator.
5244 : If the epilogue loop carries on from where the main loop left off,
5245 : it should apply the same adjustment to the final reduction result.
5246 :
5247 : If the epilogue loop can also be entered directly (rather than via
5248 : the main loop), we need to be able to handle that case in the same way,
5249 : with the same adjustment. (In principle we could add a PHI node
5250 : to select the correct adjustment, but in practice that shouldn't be
5251 : necessary.) */
5252 3947 : tree main_adjustment
5253 3947 : = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (accumulator->reduc_info);
5254 3947 : if (loop_vinfo->main_loop_edge && main_adjustment)
5255 : {
5256 3336 : gcc_assert (num_phis == 1);
5257 3336 : tree initial_value = initial_values[0];
5258 : /* Check that we can use INITIAL_VALUE as the adjustment and
5259 : initialize the accumulator with a neutral value instead. */
5260 3336 : if (!operand_equal_p (initial_value, main_adjustment))
5261 : return false;
5262 3326 : initial_values[0] = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
5263 : }
5264 3937 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info) = main_adjustment;
5265 3937 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).truncate (0);
5266 3937 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).splice (initial_values);
5267 3937 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info) = accumulator;
5268 3937 : return true;
5269 4477 : }
5270 :
5271 : /* Reduce the vector VEC_DEF down to VECTYPE with reduction operation
5272 : CODE emitting stmts before GSI. Returns a vector def of VECTYPE. */
5273 :
5274 : static tree
5275 3981 : vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
5276 : gimple_seq *seq)
5277 : {
5278 3981 : gcc_assert (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (vec_def))
5279 : || (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (vec_def)))
5280 : == MODE_VECTOR_INT));
5281 3981 : unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def)).to_constant ();
5282 3981 : unsigned nunits1 = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
5283 3981 : tree stype = TREE_TYPE (vectype);
5284 3981 : tree new_temp = vec_def;
5285 8105 : while (nunits > nunits1)
5286 : {
5287 4124 : nunits /= 2;
5288 4124 : tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
5289 4124 : stype, nunits);
5290 4124 : unsigned int bitsize = tree_to_uhwi (TYPE_SIZE (vectype1));
5291 :
5292 : /* The target has to make sure we support lowpart/highpart
5293 : extraction, either via direct vector extract or through
5294 : an integer mode punning. */
5295 4124 : tree dst1, dst2;
5296 4124 : gimple *epilog_stmt;
5297 4124 : if (can_vec_extract (TYPE_MODE (TREE_TYPE (new_temp)),
5298 8248 : TYPE_MODE (vectype1)).exists ())
5299 : {
5300 : /* Extract sub-vectors directly once vec_extract becomes
5301 : a conversion optab. */
5302 4124 : dst1 = make_ssa_name (vectype1);
5303 4124 : epilog_stmt
5304 8248 : = gimple_build_assign (dst1, BIT_FIELD_REF,
5305 : build3 (BIT_FIELD_REF, vectype1,
5306 4124 : new_temp, TYPE_SIZE (vectype1),
5307 : bitsize_int (0)));
5308 4124 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5309 4124 : dst2 = make_ssa_name (vectype1);
5310 4124 : epilog_stmt
5311 4124 : = gimple_build_assign (dst2, BIT_FIELD_REF,
5312 : build3 (BIT_FIELD_REF, vectype1,
5313 4124 : new_temp, TYPE_SIZE (vectype1),
5314 4124 : bitsize_int (bitsize)));
5315 4124 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5316 : }
5317 : else
5318 : {
5319 : /* Extract via punning to appropriately sized integer mode
5320 : vector. */
5321 0 : tree eltype = build_nonstandard_integer_type (bitsize, 1);
5322 0 : tree etype = build_vector_type (eltype, 2);
5323 0 : gcc_assert (can_vec_extract (TYPE_MODE (etype),
5324 : TYPE_MODE (eltype)).exists ());
5325 0 : tree tem = make_ssa_name (etype);
5326 0 : epilog_stmt = gimple_build_assign (tem, VIEW_CONVERT_EXPR,
5327 : build1 (VIEW_CONVERT_EXPR,
5328 : etype, new_temp));
5329 0 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5330 0 : new_temp = tem;
5331 0 : tem = make_ssa_name (eltype);
5332 0 : epilog_stmt
5333 0 : = gimple_build_assign (tem, BIT_FIELD_REF,
5334 : build3 (BIT_FIELD_REF, eltype,
5335 0 : new_temp, TYPE_SIZE (eltype),
5336 : bitsize_int (0)));
5337 0 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5338 0 : dst1 = make_ssa_name (vectype1);
5339 0 : epilog_stmt = gimple_build_assign (dst1, VIEW_CONVERT_EXPR,
5340 : build1 (VIEW_CONVERT_EXPR,
5341 : vectype1, tem));
5342 0 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5343 0 : tem = make_ssa_name (eltype);
5344 0 : epilog_stmt
5345 0 : = gimple_build_assign (tem, BIT_FIELD_REF,
5346 : build3 (BIT_FIELD_REF, eltype,
5347 0 : new_temp, TYPE_SIZE (eltype),
5348 0 : bitsize_int (bitsize)));
5349 0 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5350 0 : dst2 = make_ssa_name (vectype1);
5351 0 : epilog_stmt = gimple_build_assign (dst2, VIEW_CONVERT_EXPR,
5352 : build1 (VIEW_CONVERT_EXPR,
5353 : vectype1, tem));
5354 0 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5355 : }
5356 :
5357 4124 : new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
5358 : }
5359 3981 : if (!useless_type_conversion_p (vectype, TREE_TYPE (new_temp)))
5360 : {
5361 46 : tree dst3 = make_ssa_name (vectype);
5362 46 : gimple *epilog_stmt = gimple_build_assign (dst3, VIEW_CONVERT_EXPR,
5363 : build1 (VIEW_CONVERT_EXPR,
5364 : vectype, new_temp));
5365 46 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5366 46 : new_temp = dst3;
5367 : }
5368 :
5369 3981 : return new_temp;
5370 : }
5371 :
5372 : /* Function vect_create_epilog_for_reduction
5373 :
5374 : Create code at the loop-epilog to finalize the result of a reduction
5375 : computation.
5376 :
5377 : STMT_INFO is the scalar reduction stmt that is being vectorized.
5378 : SLP_NODE is an SLP node containing a group of reduction statements. The
5379 : first one in this group is STMT_INFO.
5380 : SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
5381 : REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
5382 : (counting from 0)
5383 : LOOP_EXIT is the edge to update in the merge block. In the case of a single
5384 : exit this edge is always the main loop exit.
5385 :
5386 : This function:
5387 : 1. Completes the reduction def-use cycles.
5388 : 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
5389 : by calling the function specified by REDUC_FN if available, or by
5390 : other means (whole-vector shifts or a scalar loop).
5391 : The function also creates a new phi node at the loop exit to preserve
5392 : loop-closed form, as illustrated below.
5393 :
5394 : The flow at the entry to this function:
5395 :
5396 : loop:
5397 : vec_def = phi <vec_init, null> # REDUCTION_PHI
5398 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5399 : s_loop = scalar_stmt # (scalar) STMT_INFO
5400 : loop_exit:
5401 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5402 : use <s_out0>
5403 : use <s_out0>
5404 :
5405 : The above is transformed by this function into:
5406 :
5407 : loop:
5408 : vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
5409 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5410 : s_loop = scalar_stmt # (scalar) STMT_INFO
5411 : loop_exit:
5412 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5413 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5414 : v_out2 = reduce <v_out1>
5415 : s_out3 = extract_field <v_out2, 0>
5416 : s_out4 = adjust_result <s_out3>
5417 : use <s_out4>
5418 : use <s_out4>
5419 : */
5420 :
5421 : static void
5422 22094 : vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
5423 : stmt_vec_info stmt_info,
5424 : slp_tree slp_node,
5425 : slp_instance slp_node_instance,
5426 : edge loop_exit)
5427 : {
5428 22094 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
5429 22094 : code_helper code = VECT_REDUC_INFO_CODE (reduc_info);
5430 22094 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
5431 22094 : tree vectype;
5432 22094 : machine_mode mode;
5433 22094 : basic_block exit_bb;
5434 22094 : gimple *new_phi = NULL, *phi = NULL;
5435 22094 : gimple_stmt_iterator exit_gsi;
5436 22094 : tree new_temp = NULL_TREE, new_name, new_scalar_dest;
5437 22094 : gimple *epilog_stmt = NULL;
5438 22094 : gimple *exit_phi;
5439 22094 : tree def;
5440 22094 : tree orig_name, scalar_result;
5441 22094 : imm_use_iterator imm_iter;
5442 22094 : use_operand_p use_p;
5443 22094 : gimple *use_stmt;
5444 22094 : auto_vec<tree> reduc_inputs;
5445 22094 : int j, i;
5446 22094 : vec<tree> &scalar_results = VECT_REDUC_INFO_SCALAR_RESULTS (reduc_info);
5447 22094 : unsigned int k;
5448 : /* SLP reduction without reduction chain, e.g.,
5449 : # a1 = phi <a2, a0>
5450 : # b1 = phi <b2, b0>
5451 : a2 = operation (a1)
5452 : b2 = operation (b1) */
5453 22094 : const bool slp_reduc = !reduc_info->is_reduc_chain;
5454 22094 : tree induction_index = NULL_TREE;
5455 :
5456 22094 : unsigned int group_size = SLP_TREE_LANES (slp_node);
5457 :
5458 22094 : bool double_reduc = false;
5459 22094 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
5460 22094 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
5461 : {
5462 0 : double_reduc = true;
5463 0 : gcc_assert (slp_reduc);
5464 : }
5465 :
5466 22094 : vectype = VECT_REDUC_INFO_VECTYPE (reduc_info);
5467 22094 : gcc_assert (vectype);
5468 22094 : mode = TYPE_MODE (vectype);
5469 :
5470 22094 : tree induc_val = NULL_TREE;
5471 22094 : tree adjustment_def = NULL;
5472 : /* Optimize: for induction condition reduction, if we can't use zero
5473 : for induc_val, use initial_def. */
5474 22094 : if (VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5475 62 : induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
5476 22032 : else if (double_reduc)
5477 : ;
5478 : else
5479 22032 : adjustment_def = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info);
5480 :
5481 22094 : stmt_vec_info single_live_out_stmt[] = { stmt_info };
5482 22094 : array_slice<const stmt_vec_info> live_out_stmts = single_live_out_stmt;
5483 22094 : if (slp_reduc)
5484 : /* All statements produce live-out values. */
5485 43726 : live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
5486 :
5487 22094 : unsigned vec_num
5488 22094 : = SLP_TREE_VEC_DEFS (slp_node_instance->reduc_phis).length ();
5489 :
5490 : /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
5491 : which is updated with the current index of the loop for every match of
5492 : the original loop's cond_expr (VEC_STMT). This results in a vector
5493 : containing the last time the condition passed for that vector lane.
5494 : The first match will be a 1 to allow 0 to be used for non-matching
5495 : indexes. If there are no matches at all then the vector will be all
5496 : zeroes.
5497 :
5498 : PR92772: This algorithm is broken for architectures that support
5499 : masked vectors, but do not provide fold_extract_last. */
5500 22094 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION)
5501 : {
5502 99 : gcc_assert (!double_reduc);
5503 99 : auto_vec<std::pair<tree, bool>, 2> ccompares;
5504 99 : slp_tree cond_node = slp_node_instance->root;
5505 207 : while (cond_node != slp_node_instance->reduc_phis)
5506 : {
5507 108 : stmt_vec_info cond_info = SLP_TREE_REPRESENTATIVE (cond_node);
5508 108 : if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
5509 : {
5510 108 : gimple *vec_stmt
5511 108 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (cond_node)[0]);
5512 108 : gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
5513 108 : ccompares.safe_push
5514 108 : (std::make_pair (gimple_assign_rhs1 (vec_stmt),
5515 108 : SLP_TREE_REDUC_IDX (cond_node) == 2));
5516 : }
5517 108 : int slp_reduc_idx = SLP_TREE_REDUC_IDX (cond_node);
5518 108 : cond_node = SLP_TREE_CHILDREN (cond_node)[slp_reduc_idx];
5519 : }
5520 99 : gcc_assert (ccompares.length () != 0);
5521 :
5522 99 : tree indx_before_incr, indx_after_incr;
5523 99 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
5524 99 : int scalar_precision
5525 99 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype)));
5526 99 : tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
5527 99 : tree cr_index_vector_type = get_related_vectype_for_scalar_type
5528 99 : (TYPE_MODE (vectype), cr_index_scalar_type,
5529 : TYPE_VECTOR_SUBPARTS (vectype));
5530 :
5531 : /* First we create a simple vector induction variable which starts
5532 : with the values {1,2,3,...} (SERIES_VECT) and increments by the
5533 : vector size (STEP). */
5534 :
5535 : /* Create a {1,2,3,...} vector. */
5536 99 : tree series_vect = build_index_vector (cr_index_vector_type, 1, 1);
5537 :
5538 : /* Create a vector of the step value. */
5539 99 : tree step = build_int_cst (cr_index_scalar_type, nunits_out);
5540 99 : tree vec_step = build_vector_from_val (cr_index_vector_type, step);
5541 :
5542 : /* Create an induction variable. */
5543 99 : gimple_stmt_iterator incr_gsi;
5544 99 : bool insert_after;
5545 99 : vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
5546 : &incr_gsi, &insert_after);
5547 99 : create_iv (series_vect, PLUS_EXPR, vec_step, NULL_TREE, loop, &incr_gsi,
5548 : insert_after, &indx_before_incr, &indx_after_incr);
5549 :
5550 : /* Next create a new phi node vector (NEW_PHI_TREE) which starts
5551 : filled with zeros (VEC_ZERO). */
5552 :
5553 : /* Create a vector of 0s. */
5554 99 : tree zero = build_zero_cst (cr_index_scalar_type);
5555 99 : tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
5556 :
5557 : /* Create a vector phi node. */
5558 99 : tree new_phi_tree = make_ssa_name (cr_index_vector_type);
5559 99 : new_phi = create_phi_node (new_phi_tree, loop->header);
5560 99 : add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
5561 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
5562 :
5563 : /* Now take the condition from the loops original cond_exprs
5564 : and produce a new cond_exprs (INDEX_COND_EXPR) which for
5565 : every match uses values from the induction variable
5566 : (INDEX_BEFORE_INCR) otherwise uses values from the phi node
5567 : (NEW_PHI_TREE).
5568 : Finally, we update the phi (NEW_PHI_TREE) to take the value of
5569 : the new cond_expr (INDEX_COND_EXPR). */
5570 99 : gimple_seq stmts = NULL;
5571 306 : for (int i = ccompares.length () - 1; i != -1; --i)
5572 : {
5573 108 : tree ccompare = ccompares[i].first;
5574 108 : if (ccompares[i].second)
5575 81 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5576 : cr_index_vector_type,
5577 : ccompare,
5578 : indx_before_incr, new_phi_tree);
5579 : else
5580 27 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5581 : cr_index_vector_type,
5582 : ccompare,
5583 : new_phi_tree, indx_before_incr);
5584 : }
5585 99 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
5586 :
5587 : /* Update the phi with the vec cond. */
5588 99 : induction_index = new_phi_tree;
5589 99 : add_phi_arg (as_a <gphi *> (new_phi), induction_index,
5590 : loop_latch_edge (loop), UNKNOWN_LOCATION);
5591 99 : }
5592 :
5593 : /* 2. Create epilog code.
5594 : The reduction epilog code operates across the elements of the vector
5595 : of partial results computed by the vectorized loop.
5596 : The reduction epilog code consists of:
5597 :
5598 : step 1: compute the scalar result in a vector (v_out2)
5599 : step 2: extract the scalar result (s_out3) from the vector (v_out2)
5600 : step 3: adjust the scalar result (s_out3) if needed.
5601 :
5602 : Step 1 can be accomplished using one the following three schemes:
5603 : (scheme 1) using reduc_fn, if available.
5604 : (scheme 2) using whole-vector shifts, if available.
5605 : (scheme 3) using a scalar loop. In this case steps 1+2 above are
5606 : combined.
5607 :
5608 : The overall epilog code looks like this:
5609 :
5610 : s_out0 = phi <s_loop> # original EXIT_PHI
5611 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5612 : v_out2 = reduce <v_out1> # step 1
5613 : s_out3 = extract_field <v_out2, 0> # step 2
5614 : s_out4 = adjust_result <s_out3> # step 3
5615 :
5616 : (step 3 is optional, and steps 1 and 2 may be combined).
5617 : Lastly, the uses of s_out0 are replaced by s_out4. */
5618 :
5619 :
5620 : /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
5621 : v_out1 = phi <VECT_DEF>
5622 : Store them in NEW_PHIS. */
5623 : /* We need to reduce values in all exits. */
5624 22094 : exit_bb = loop_exit->dest;
5625 22094 : exit_gsi = gsi_after_labels (exit_bb);
5626 22094 : reduc_inputs.create (vec_num);
5627 67876 : for (unsigned i = 0; i < vec_num; i++)
5628 : {
5629 23688 : gimple_seq stmts = NULL;
5630 23688 : def = vect_get_slp_vect_def (slp_node, i);
5631 23688 : tree new_def = copy_ssa_name (def);
5632 23688 : phi = create_phi_node (new_def, exit_bb);
5633 23688 : if (LOOP_VINFO_MAIN_EXIT (loop_vinfo) == loop_exit)
5634 23661 : SET_PHI_ARG_DEF (phi, loop_exit->dest_idx, def);
5635 : else
5636 : {
5637 57 : for (unsigned k = 0; k < gimple_phi_num_args (phi); k++)
5638 30 : SET_PHI_ARG_DEF (phi, k, def);
5639 : }
5640 23688 : new_def = gimple_convert (&stmts, vectype, new_def);
5641 23688 : reduc_inputs.quick_push (new_def);
5642 23688 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5643 : }
5644 :
5645 : /* 2.2 Get the original scalar reduction variable as defined in the loop.
5646 : In case STMT is a "pattern-stmt" (i.e. - it represents a reduction
5647 : pattern), the scalar-def is taken from the original stmt that the
5648 : pattern-stmt (STMT) replaces. */
5649 :
5650 22938 : tree scalar_dest = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5651 22094 : tree scalar_type = TREE_TYPE (scalar_dest);
5652 22094 : scalar_results.truncate (0);
5653 22094 : scalar_results.reserve_exact (group_size);
5654 22094 : new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
5655 :
5656 : /* True if we should implement SLP_REDUC using native reduction operations
5657 : instead of scalar operations. */
5658 22094 : const bool direct_slp_reduc
5659 22094 : = (reduc_fn != IFN_LAST
5660 22094 : && slp_reduc
5661 22094 : && !TYPE_VECTOR_SUBPARTS (vectype).is_constant ());
5662 :
5663 : /* If signed overflow is undefined we might need to perform reduction
5664 : computations in an unsigned type. */
5665 22094 : tree compute_vectype = vectype;
5666 22094 : if (ANY_INTEGRAL_TYPE_P (vectype)
5667 14961 : && TYPE_OVERFLOW_UNDEFINED (vectype)
5668 5512 : && code.is_tree_code ()
5669 27606 : && arith_code_with_undefined_signed_overflow ((tree_code) code))
5670 4076 : compute_vectype = unsigned_type_for (vectype);
5671 :
5672 : /* In case of reduction chain, e.g.,
5673 : # a1 = phi <a3, a0>
5674 : a2 = operation (a1)
5675 : a3 = operation (a2),
5676 :
5677 : we may end up with more than one vector result. Here we reduce them
5678 : to one vector.
5679 :
5680 : The same is true for a SLP reduction, e.g.,
5681 : # a1 = phi <a2, a0>
5682 : # b1 = phi <b2, b0>
5683 : a2 = operation (a1)
5684 : b2 = operation (a2),
5685 :
5686 : where we can end up with more than one vector as well. We can
5687 : easily accumulate vectors when the number of vector elements is
5688 : a multiple of the SLP group size.
5689 :
5690 : The same is true if we couldn't use a single defuse cycle. */
5691 22094 : if ((!slp_reduc
5692 : || direct_slp_reduc
5693 : || (slp_reduc
5694 22094 : && constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype), group_size)))
5695 44188 : && reduc_inputs.length () > 1)
5696 : {
5697 550 : gimple_seq stmts = NULL;
5698 550 : tree single_input = reduc_inputs[0];
5699 550 : if (compute_vectype != vectype)
5700 167 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5701 : compute_vectype, single_input);
5702 1989 : for (k = 1; k < reduc_inputs.length (); k++)
5703 : {
5704 1439 : tree input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5705 1439 : compute_vectype, reduc_inputs[k]);
5706 1439 : single_input = gimple_build (&stmts, code, compute_vectype,
5707 : single_input, input);
5708 : }
5709 550 : if (compute_vectype != vectype)
5710 167 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5711 : vectype, single_input);
5712 550 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5713 :
5714 550 : reduc_inputs.truncate (0);
5715 550 : reduc_inputs.safe_push (single_input);
5716 : }
5717 :
5718 22094 : tree orig_reduc_input = reduc_inputs[0];
5719 :
5720 : /* If this loop is an epilogue loop that can be skipped after the
5721 : main loop, we can only share a reduction operation between the
5722 : main loop and the epilogue if we put it at the target of the
5723 : skip edge.
5724 :
5725 : We can still reuse accumulators if this check fails. Doing so has
5726 : the minor(?) benefit of making the epilogue loop's scalar result
5727 : independent of the main loop's scalar result. */
5728 22094 : bool unify_with_main_loop_p = false;
5729 22094 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
5730 3937 : && loop_vinfo->skip_this_loop_edge
5731 3771 : && single_succ_p (exit_bb)
5732 22111 : && single_succ (exit_bb) == loop_vinfo->skip_this_loop_edge->dest)
5733 : {
5734 17 : unify_with_main_loop_p = true;
5735 :
5736 17 : basic_block reduc_block = loop_vinfo->skip_this_loop_edge->dest;
5737 17 : reduc_inputs[0] = make_ssa_name (vectype);
5738 17 : gphi *new_phi = create_phi_node (reduc_inputs[0], reduc_block);
5739 17 : add_phi_arg (new_phi, orig_reduc_input, single_succ_edge (exit_bb),
5740 : UNKNOWN_LOCATION);
5741 17 : add_phi_arg (new_phi,
5742 17 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)->reduc_input,
5743 : loop_vinfo->skip_this_loop_edge, UNKNOWN_LOCATION);
5744 17 : exit_gsi = gsi_after_labels (reduc_block);
5745 : }
5746 :
5747 : /* Shouldn't be used beyond this point. */
5748 22094 : exit_bb = nullptr;
5749 :
5750 : /* If we are operating on a mask vector and do not support direct mask
5751 : reduction, work on a bool data vector instead of a mask vector. */
5752 22094 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5753 235 : && VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info)
5754 22279 : && vectype != VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info))
5755 : {
5756 185 : compute_vectype = vectype = VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info);
5757 185 : gimple_seq stmts = NULL;
5758 378 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
5759 386 : reduc_inputs[i] = gimple_build (&stmts, VEC_COND_EXPR, vectype,
5760 193 : reduc_inputs[i],
5761 : build_one_cst (vectype),
5762 : build_zero_cst (vectype));
5763 185 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5764 : }
5765 :
5766 22094 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5767 99 : && reduc_fn != IFN_LAST)
5768 : {
5769 : /* For condition reductions, we have a vector (REDUC_INPUTS 0) containing
5770 : various data values where the condition matched and another vector
5771 : (INDUCTION_INDEX) containing all the indexes of those matches. We
5772 : need to extract the last matching index (which will be the index with
5773 : highest value) and use this to index into the data vector.
5774 : For the case where there were no matches, the data vector will contain
5775 : all default values and the index vector will be all zeros. */
5776 :
5777 : /* Get various versions of the type of the vector of indexes. */
5778 14 : tree index_vec_type = TREE_TYPE (induction_index);
5779 14 : gcc_checking_assert (TYPE_UNSIGNED (index_vec_type));
5780 14 : tree index_scalar_type = TREE_TYPE (index_vec_type);
5781 14 : tree index_vec_cmp_type = truth_type_for (index_vec_type);
5782 :
5783 : /* Get an unsigned integer version of the type of the data vector. */
5784 14 : int scalar_precision
5785 14 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
5786 14 : tree scalar_type_unsigned = make_unsigned_type (scalar_precision);
5787 14 : tree vectype_unsigned = get_same_sized_vectype (scalar_type_unsigned,
5788 : vectype);
5789 :
5790 : /* First we need to create a vector (ZERO_VEC) of zeros and another
5791 : vector (MAX_INDEX_VEC) filled with the last matching index, which we
5792 : can create using a MAX reduction and then expanding.
5793 : In the case where the loop never made any matches, the max index will
5794 : be zero. */
5795 :
5796 : /* Vector of {0, 0, 0,...}. */
5797 14 : tree zero_vec = build_zero_cst (vectype);
5798 :
5799 : /* Find maximum value from the vector of found indexes. */
5800 14 : tree max_index = make_ssa_name (index_scalar_type);
5801 14 : gcall *max_index_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5802 : 1, induction_index);
5803 14 : gimple_call_set_lhs (max_index_stmt, max_index);
5804 14 : gsi_insert_before (&exit_gsi, max_index_stmt, GSI_SAME_STMT);
5805 :
5806 : /* Vector of {max_index, max_index, max_index,...}. */
5807 14 : tree max_index_vec = make_ssa_name (index_vec_type);
5808 14 : tree max_index_vec_rhs = build_vector_from_val (index_vec_type,
5809 : max_index);
5810 14 : gimple *max_index_vec_stmt = gimple_build_assign (max_index_vec,
5811 : max_index_vec_rhs);
5812 14 : gsi_insert_before (&exit_gsi, max_index_vec_stmt, GSI_SAME_STMT);
5813 :
5814 : /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
5815 : with the vector (INDUCTION_INDEX) of found indexes, choosing values
5816 : from the data vector (REDUC_INPUTS 0) for matches, 0 (ZERO_VEC)
5817 : otherwise. Only one value should match, resulting in a vector
5818 : (VEC_COND) with one data value and the rest zeros.
5819 : In the case where the loop never made any matches, every index will
5820 : match, resulting in a vector with all data values (which will all be
5821 : the default value). */
5822 :
5823 : /* Compare the max index vector to the vector of found indexes to find
5824 : the position of the max value. */
5825 14 : tree vec_compare = make_ssa_name (index_vec_cmp_type);
5826 14 : gimple *vec_compare_stmt = gimple_build_assign (vec_compare, EQ_EXPR,
5827 : induction_index,
5828 : max_index_vec);
5829 14 : gsi_insert_before (&exit_gsi, vec_compare_stmt, GSI_SAME_STMT);
5830 :
5831 : /* Use the compare to choose either values from the data vector or
5832 : zero. */
5833 14 : tree vec_cond = make_ssa_name (vectype);
5834 14 : gimple *vec_cond_stmt = gimple_build_assign (vec_cond, VEC_COND_EXPR,
5835 : vec_compare,
5836 14 : reduc_inputs[0],
5837 : zero_vec);
5838 14 : gsi_insert_before (&exit_gsi, vec_cond_stmt, GSI_SAME_STMT);
5839 :
5840 : /* Finally we need to extract the data value from the vector (VEC_COND)
5841 : into a scalar (MATCHED_DATA_REDUC). Logically we want to do a OR
5842 : reduction, but because this doesn't exist, we can use a MAX reduction
5843 : instead. The data value might be signed or a float so we need to cast
5844 : it first.
5845 : In the case where the loop never made any matches, the data values are
5846 : all identical, and so will reduce down correctly. */
5847 :
5848 : /* Make the matched data values unsigned. */
5849 14 : tree vec_cond_cast = make_ssa_name (vectype_unsigned);
5850 14 : tree vec_cond_cast_rhs = build1 (VIEW_CONVERT_EXPR, vectype_unsigned,
5851 : vec_cond);
5852 14 : gimple *vec_cond_cast_stmt = gimple_build_assign (vec_cond_cast,
5853 : VIEW_CONVERT_EXPR,
5854 : vec_cond_cast_rhs);
5855 14 : gsi_insert_before (&exit_gsi, vec_cond_cast_stmt, GSI_SAME_STMT);
5856 :
5857 : /* Reduce down to a scalar value. */
5858 14 : tree data_reduc = make_ssa_name (scalar_type_unsigned);
5859 14 : gcall *data_reduc_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5860 : 1, vec_cond_cast);
5861 14 : gimple_call_set_lhs (data_reduc_stmt, data_reduc);
5862 14 : gsi_insert_before (&exit_gsi, data_reduc_stmt, GSI_SAME_STMT);
5863 :
5864 : /* Convert the reduced value back to the result type and set as the
5865 : result. */
5866 14 : gimple_seq stmts = NULL;
5867 14 : new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR, scalar_type,
5868 : data_reduc);
5869 14 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5870 14 : scalar_results.safe_push (new_temp);
5871 14 : }
5872 22080 : else if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5873 85 : && reduc_fn == IFN_LAST)
5874 : {
5875 : /* Condition reduction without supported IFN_REDUC_MAX. Generate
5876 : idx = 0;
5877 : idx_val = induction_index[0];
5878 : val = data_reduc[0];
5879 : for (idx = 0, val = init, i = 0; i < nelts; ++i)
5880 : if (induction_index[i] > idx_val)
5881 : val = data_reduc[i], idx_val = induction_index[i];
5882 : return val; */
5883 :
5884 85 : tree data_eltype = TREE_TYPE (vectype);
5885 85 : tree idx_eltype = TREE_TYPE (TREE_TYPE (induction_index));
5886 85 : unsigned HOST_WIDE_INT el_size = tree_to_uhwi (TYPE_SIZE (idx_eltype));
5887 85 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index));
5888 : /* Enforced by vectorizable_reduction, which ensures we have target
5889 : support before allowing a conditional reduction on variable-length
5890 : vectors. */
5891 85 : unsigned HOST_WIDE_INT v_size = el_size * nunits.to_constant ();
5892 85 : tree idx_val = NULL_TREE, val = NULL_TREE;
5893 511 : for (unsigned HOST_WIDE_INT off = 0; off < v_size; off += el_size)
5894 : {
5895 426 : tree old_idx_val = idx_val;
5896 426 : tree old_val = val;
5897 426 : idx_val = make_ssa_name (idx_eltype);
5898 426 : epilog_stmt = gimple_build_assign (idx_val, BIT_FIELD_REF,
5899 : build3 (BIT_FIELD_REF, idx_eltype,
5900 : induction_index,
5901 426 : bitsize_int (el_size),
5902 426 : bitsize_int (off)));
5903 426 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5904 426 : val = make_ssa_name (data_eltype);
5905 852 : epilog_stmt = gimple_build_assign (val, BIT_FIELD_REF,
5906 : build3 (BIT_FIELD_REF,
5907 : data_eltype,
5908 426 : reduc_inputs[0],
5909 426 : bitsize_int (el_size),
5910 426 : bitsize_int (off)));
5911 426 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5912 426 : if (off != 0)
5913 : {
5914 341 : tree new_idx_val = idx_val;
5915 341 : if (off != v_size - el_size)
5916 : {
5917 256 : new_idx_val = make_ssa_name (idx_eltype);
5918 256 : epilog_stmt = gimple_build_assign (new_idx_val,
5919 : MAX_EXPR, idx_val,
5920 : old_idx_val);
5921 256 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5922 : }
5923 341 : tree cond = make_ssa_name (boolean_type_node);
5924 341 : epilog_stmt = gimple_build_assign (cond, GT_EXPR,
5925 : idx_val, old_idx_val);
5926 341 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5927 341 : tree new_val = make_ssa_name (data_eltype);
5928 341 : epilog_stmt = gimple_build_assign (new_val, COND_EXPR,
5929 : cond, val, old_val);
5930 341 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5931 341 : idx_val = new_idx_val;
5932 341 : val = new_val;
5933 : }
5934 : }
5935 : /* Convert the reduced value back to the result type and set as the
5936 : result. */
5937 85 : gimple_seq stmts = NULL;
5938 85 : val = gimple_convert (&stmts, scalar_type, val);
5939 85 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5940 85 : scalar_results.safe_push (val);
5941 85 : }
5942 :
5943 : /* 2.3 Create the reduction code, using one of the three schemes described
5944 : above. In SLP we simply need to extract all the elements from the
5945 : vector (without reducing them), so we use scalar shifts. */
5946 21995 : else if (reduc_fn != IFN_LAST && (!slp_reduc || group_size == 1))
5947 : {
5948 20178 : tree tmp;
5949 20178 : tree vec_elem_type;
5950 :
5951 : /* Case 1: Create:
5952 : v_out2 = reduc_expr <v_out1> */
5953 :
5954 20178 : if (dump_enabled_p ())
5955 1552 : dump_printf_loc (MSG_NOTE, vect_location,
5956 : "Reduce using direct vector reduction.\n");
5957 :
5958 20178 : gimple_seq stmts = NULL;
5959 20178 : vec_elem_type = TREE_TYPE (vectype);
5960 20178 : new_temp = gimple_build (&stmts, as_combined_fn (reduc_fn),
5961 20178 : vec_elem_type, reduc_inputs[0]);
5962 20178 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
5963 20178 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5964 :
5965 20178 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5966 62 : && induc_val)
5967 : {
5968 : /* Earlier we set the initial value to be a vector if induc_val
5969 : values. Check the result and if it is induc_val then replace
5970 : with the original initial value, unless induc_val is
5971 : the same as initial_def already. */
5972 60 : tree zcompare = make_ssa_name (boolean_type_node);
5973 60 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
5974 : new_temp, induc_val);
5975 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5976 60 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
5977 60 : tmp = make_ssa_name (new_scalar_dest);
5978 60 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
5979 : initial_def, new_temp);
5980 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5981 60 : new_temp = tmp;
5982 : }
5983 :
5984 20178 : scalar_results.safe_push (new_temp);
5985 20178 : }
5986 1619 : else if (direct_slp_reduc)
5987 : {
5988 : /* Here we create one vector for each of the GROUP_SIZE results,
5989 : with the elements for other SLP statements replaced with the
5990 : neutral value. We can then do a normal reduction on each vector. */
5991 :
5992 : /* Enforced by vectorizable_reduction. */
5993 : gcc_assert (reduc_inputs.length () == 1);
5994 : gcc_assert (pow2p_hwi (group_size));
5995 :
5996 : gimple_seq seq = NULL;
5997 :
5998 : /* Build a vector {0, 1, 2, ...}, with the same number of elements
5999 : and the same element size as VECTYPE. */
6000 : tree index = build_index_vector (vectype, 0, 1);
6001 : tree index_type = TREE_TYPE (index);
6002 : tree index_elt_type = TREE_TYPE (index_type);
6003 : tree mask_type = truth_type_for (index_type);
6004 :
6005 : /* Create a vector that, for each element, identifies which of
6006 : the results should use it. */
6007 : tree index_mask = build_int_cst (index_elt_type, group_size - 1);
6008 : index = gimple_build (&seq, BIT_AND_EXPR, index_type, index,
6009 : build_vector_from_val (index_type, index_mask));
6010 :
6011 : /* Get a neutral vector value. This is simply a splat of the neutral
6012 : scalar value if we have one, otherwise the initial scalar value
6013 : is itself a neutral value. */
6014 : tree vector_identity = NULL_TREE;
6015 : tree neutral_op = neutral_op_for_reduction (TREE_TYPE (vectype), code,
6016 : NULL_TREE, false);
6017 : if (neutral_op)
6018 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
6019 : neutral_op);
6020 : for (unsigned int i = 0; i < group_size; ++i)
6021 : {
6022 : /* If there's no universal neutral value, we can use the
6023 : initial scalar value from the original PHI. This is used
6024 : for MIN and MAX reduction, for example. */
6025 : if (!neutral_op)
6026 : {
6027 : tree scalar_value
6028 : = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[i];
6029 : scalar_value = gimple_convert (&seq, TREE_TYPE (vectype),
6030 : scalar_value);
6031 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
6032 : scalar_value);
6033 : }
6034 :
6035 : /* Calculate the equivalent of:
6036 :
6037 : sel[j] = (index[j] == i);
6038 :
6039 : which selects the elements of REDUC_INPUTS[0] that should
6040 : be included in the result. */
6041 : tree compare_val = build_int_cst (index_elt_type, i);
6042 : compare_val = build_vector_from_val (index_type, compare_val);
6043 : tree sel = gimple_build (&seq, EQ_EXPR, mask_type,
6044 : index, compare_val);
6045 :
6046 : /* Calculate the equivalent of:
6047 :
6048 : vec = seq ? reduc_inputs[0] : vector_identity;
6049 :
6050 : VEC is now suitable for a full vector reduction. */
6051 : tree vec = gimple_build (&seq, VEC_COND_EXPR, vectype,
6052 : sel, reduc_inputs[0], vector_identity);
6053 :
6054 : /* Do the reduction and convert it to the appropriate type. */
6055 : tree scalar = gimple_build (&seq, as_combined_fn (reduc_fn),
6056 : TREE_TYPE (vectype), vec);
6057 : scalar = gimple_convert (&seq, scalar_type, scalar);
6058 : scalar_results.safe_push (scalar);
6059 : }
6060 : gsi_insert_seq_before (&exit_gsi, seq, GSI_SAME_STMT);
6061 : }
6062 : else
6063 : {
6064 1619 : bool reduce_with_shift;
6065 1619 : tree vec_temp;
6066 :
6067 1619 : gcc_assert (slp_reduc || reduc_inputs.length () == 1);
6068 :
6069 : /* See if the target wants to do the final (shift) reduction
6070 : in a vector mode of smaller size and first reduce upper/lower
6071 : halves against each other. */
6072 1817 : enum machine_mode mode1 = mode;
6073 1817 : tree stype = TREE_TYPE (vectype);
6074 1817 : if (compute_vectype != vectype)
6075 : {
6076 459 : stype = unsigned_type_for (stype);
6077 459 : gimple_seq stmts = NULL;
6078 976 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
6079 : {
6080 517 : tree new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR,
6081 517 : compute_vectype, reduc_inputs[i]);
6082 517 : reduc_inputs[i] = new_temp;
6083 : }
6084 459 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6085 : }
6086 1817 : unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
6087 1817 : unsigned nunits1 = nunits;
6088 1817 : if ((mode1 = targetm.vectorize.split_reduction (mode)) != mode
6089 1817 : && reduc_inputs.length () == 1)
6090 : {
6091 41 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
6092 : /* For SLP reductions we have to make sure lanes match up, but
6093 : since we're doing individual element final reduction reducing
6094 : vector width here is even more important.
6095 : ??? We can also separate lanes with permutes, for the common
6096 : case of power-of-two group-size odd/even extracts would work. */
6097 41 : if (slp_reduc && nunits != nunits1)
6098 : {
6099 41 : nunits1 = least_common_multiple (nunits1, group_size);
6100 82 : gcc_assert (exact_log2 (nunits1) != -1 && nunits1 <= nunits);
6101 : }
6102 : }
6103 1776 : else if (!slp_reduc
6104 1776 : && (mode1 = targetm.vectorize.split_reduction (mode)) != mode)
6105 0 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
6106 :
6107 1817 : tree vectype1 = compute_vectype;
6108 1817 : if (mode1 != mode)
6109 : {
6110 47 : vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
6111 47 : stype, nunits1);
6112 : /* First reduce the vector to the desired vector size we should
6113 : do shift reduction on by combining upper and lower halves. */
6114 47 : gimple_seq stmts = NULL;
6115 47 : new_temp = vect_create_partial_epilog (reduc_inputs[0], vectype1,
6116 : code, &stmts);
6117 47 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6118 47 : reduc_inputs[0] = new_temp;
6119 : }
6120 :
6121 1817 : reduce_with_shift = have_whole_vector_shift (mode1);
6122 754 : if (!VECTOR_MODE_P (mode1)
6123 2569 : || !directly_supported_p (code, vectype1))
6124 : reduce_with_shift = false;
6125 :
6126 1800 : if (reduce_with_shift && (!slp_reduc || group_size == 1))
6127 : {
6128 1549 : int element_bitsize = vector_element_bits (vectype1);
6129 : /* Enforced by vectorizable_reduction, which disallows SLP reductions
6130 : for variable-length vectors and also requires direct target support
6131 : for loop reductions. */
6132 1549 : int nelements = TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6133 1549 : vec_perm_builder sel;
6134 1549 : vec_perm_indices indices;
6135 :
6136 1549 : int elt_offset;
6137 :
6138 1549 : tree zero_vec = build_zero_cst (vectype1);
6139 : /* Case 2: Create:
6140 : for (offset = nelements/2; offset >= 1; offset/=2)
6141 : {
6142 : Create: va' = vec_shift <va, offset>
6143 : Create: va = vop <va, va'>
6144 : } */
6145 :
6146 1549 : if (dump_enabled_p ())
6147 384 : dump_printf_loc (MSG_NOTE, vect_location,
6148 : "Reduce using vector shifts\n");
6149 :
6150 1549 : gimple_seq stmts = NULL;
6151 1549 : new_temp = gimple_convert (&stmts, vectype1, reduc_inputs[0]);
6152 1549 : for (elt_offset = nelements / 2;
6153 3401 : elt_offset >= 1;
6154 1852 : elt_offset /= 2)
6155 : {
6156 1852 : calc_vec_perm_mask_for_shift (elt_offset, nelements, &sel);
6157 1852 : indices.new_vector (sel, 2, nelements);
6158 1852 : tree mask = vect_gen_perm_mask_any (vectype1, indices);
6159 1852 : new_name = gimple_build (&stmts, VEC_PERM_EXPR, vectype1,
6160 : new_temp, zero_vec, mask);
6161 1852 : new_temp = gimple_build (&stmts, code,
6162 : vectype1, new_name, new_temp);
6163 : }
6164 :
6165 : /* 2.4 Extract the final scalar result. Create:
6166 : s_out3 = extract_field <v_out2, bitpos> */
6167 :
6168 1549 : if (dump_enabled_p ())
6169 384 : dump_printf_loc (MSG_NOTE, vect_location,
6170 : "extract scalar result\n");
6171 :
6172 1549 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype1),
6173 1549 : new_temp, bitsize_int (element_bitsize),
6174 1549 : bitsize_zero_node);
6175 1549 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6176 1549 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6177 1549 : scalar_results.safe_push (new_temp);
6178 1549 : }
6179 : else
6180 : {
6181 : /* Case 3: Create:
6182 : s = extract_field <v_out2, 0>
6183 : for (offset = element_size;
6184 : offset < vector_size;
6185 : offset += element_size;)
6186 : {
6187 : Create: s' = extract_field <v_out2, offset>
6188 : Create: s = op <s, s'> // For non SLP cases
6189 : } */
6190 :
6191 268 : if (dump_enabled_p ())
6192 143 : dump_printf_loc (MSG_NOTE, vect_location,
6193 : "Reduce using scalar code.\n");
6194 :
6195 268 : tree compute_type = TREE_TYPE (vectype1);
6196 268 : unsigned element_bitsize = vector_element_bits (vectype1);
6197 268 : unsigned vec_size_in_bits = element_bitsize
6198 268 : * TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6199 268 : tree bitsize = bitsize_int (element_bitsize);
6200 268 : gimple_seq stmts = NULL;
6201 691 : FOR_EACH_VEC_ELT (reduc_inputs, i, vec_temp)
6202 : {
6203 423 : unsigned bit_offset;
6204 846 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, compute_type,
6205 423 : vec_temp, bitsize, bitsize_zero_node);
6206 :
6207 : /* In SLP we don't need to apply reduction operation, so we just
6208 : collect s' values in SCALAR_RESULTS. */
6209 423 : if (slp_reduc)
6210 413 : scalar_results.safe_push (new_temp);
6211 :
6212 423 : for (bit_offset = element_bitsize;
6213 1422 : bit_offset < vec_size_in_bits;
6214 999 : bit_offset += element_bitsize)
6215 : {
6216 999 : tree bitpos = bitsize_int (bit_offset);
6217 999 : new_name = gimple_build (&stmts, BIT_FIELD_REF,
6218 : compute_type, vec_temp,
6219 : bitsize, bitpos);
6220 999 : if (slp_reduc)
6221 : {
6222 : /* In SLP we don't need to apply reduction operation, so
6223 : we just collect s' values in SCALAR_RESULTS. */
6224 989 : new_temp = new_name;
6225 989 : scalar_results.safe_push (new_name);
6226 : }
6227 : else
6228 10 : new_temp = gimple_build (&stmts, code, compute_type,
6229 : new_name, new_temp);
6230 : }
6231 : }
6232 :
6233 : /* The only case where we need to reduce scalar results in a SLP
6234 : reduction, is unrolling. If the size of SCALAR_RESULTS is
6235 : greater than GROUP_SIZE, we reduce them combining elements modulo
6236 : GROUP_SIZE. */
6237 268 : if (slp_reduc)
6238 : {
6239 258 : tree res, first_res, new_res;
6240 :
6241 : /* Reduce multiple scalar results in case of SLP unrolling. */
6242 920 : for (j = group_size; scalar_results.iterate (j, &res);
6243 : j++)
6244 : {
6245 662 : first_res = scalar_results[j % group_size];
6246 662 : new_res = gimple_build (&stmts, code, compute_type,
6247 : first_res, res);
6248 662 : scalar_results[j % group_size] = new_res;
6249 : }
6250 258 : scalar_results.truncate (group_size);
6251 1256 : for (k = 0; k < group_size; k++)
6252 1480 : scalar_results[k] = gimple_convert (&stmts, scalar_type,
6253 740 : scalar_results[k]);
6254 : }
6255 : else
6256 : {
6257 : /* Reduction chain - we have one scalar to keep in
6258 : SCALAR_RESULTS. */
6259 10 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6260 10 : scalar_results.safe_push (new_temp);
6261 : }
6262 :
6263 268 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6264 : }
6265 :
6266 1817 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
6267 0 : && induc_val)
6268 : {
6269 : /* Earlier we set the initial value to be a vector if induc_val
6270 : values. Check the result and if it is induc_val then replace
6271 : with the original initial value, unless induc_val is
6272 : the same as initial_def already. */
6273 0 : tree zcompare = make_ssa_name (boolean_type_node);
6274 0 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
6275 0 : scalar_results[0], induc_val);
6276 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6277 0 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
6278 0 : tree tmp = make_ssa_name (new_scalar_dest);
6279 0 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
6280 0 : initial_def, scalar_results[0]);
6281 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6282 0 : scalar_results[0] = tmp;
6283 : }
6284 : }
6285 :
6286 : /* 2.5 Adjust the final result by the initial value of the reduction
6287 : variable. (When such adjustment is not needed, then
6288 : 'adjustment_def' is zero). For example, if code is PLUS we create:
6289 : new_temp = loop_exit_def + adjustment_def */
6290 :
6291 22094 : if (adjustment_def)
6292 : {
6293 15570 : gcc_assert (!slp_reduc || group_size == 1);
6294 15570 : gimple_seq stmts = NULL;
6295 15570 : if (double_reduc)
6296 : {
6297 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def)));
6298 0 : adjustment_def = gimple_convert (&stmts, vectype, adjustment_def);
6299 0 : new_temp = gimple_build (&stmts, code, vectype,
6300 0 : reduc_inputs[0], adjustment_def);
6301 : }
6302 : else
6303 : {
6304 15570 : new_temp = scalar_results[0];
6305 15570 : gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
6306 15570 : adjustment_def = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6307 : adjustment_def);
6308 15570 : new_temp = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6309 : new_temp);
6310 15570 : new_temp = gimple_build (&stmts, code, TREE_TYPE (compute_vectype),
6311 : new_temp, adjustment_def);
6312 15570 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6313 : }
6314 :
6315 15570 : epilog_stmt = gimple_seq_last_stmt (stmts);
6316 15570 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6317 15570 : scalar_results[0] = new_temp;
6318 : }
6319 :
6320 : /* Record this operation if it could be reused by the epilogue loop. */
6321 22094 : if (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION
6322 22094 : && reduc_inputs.length () == 1)
6323 21880 : loop_vinfo->reusable_accumulators.put (scalar_results[0],
6324 : { orig_reduc_input, reduc_info });
6325 :
6326 : /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
6327 : phis with new adjusted scalar results, i.e., replace use <s_out0>
6328 : with use <s_out4>.
6329 :
6330 : Transform:
6331 : loop_exit:
6332 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6333 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6334 : v_out2 = reduce <v_out1>
6335 : s_out3 = extract_field <v_out2, 0>
6336 : s_out4 = adjust_result <s_out3>
6337 : use <s_out0>
6338 : use <s_out0>
6339 :
6340 : into:
6341 :
6342 : loop_exit:
6343 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6344 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6345 : v_out2 = reduce <v_out1>
6346 : s_out3 = extract_field <v_out2, 0>
6347 : s_out4 = adjust_result <s_out3>
6348 : use <s_out4>
6349 : use <s_out4> */
6350 :
6351 44188 : gcc_assert (live_out_stmts.size () == scalar_results.length ());
6352 22094 : auto_vec<gimple *> phis;
6353 44670 : for (k = 0; k < live_out_stmts.size (); k++)
6354 : {
6355 22576 : stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
6356 22576 : tree scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
6357 :
6358 : /* Find the loop-closed-use at the loop exit of the original scalar
6359 : result. (The reduction result is expected to have two immediate uses,
6360 : one at the latch block, and one at the loop exit). Note with
6361 : early break we can have two exit blocks, so pick the correct PHI. */
6362 91521 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
6363 68945 : if (!is_gimple_debug (USE_STMT (use_p))
6364 68945 : && !flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
6365 : {
6366 22569 : gcc_assert (is_a <gphi *> (USE_STMT (use_p)));
6367 22569 : if (gimple_bb (USE_STMT (use_p)) == loop_exit->dest)
6368 22561 : phis.safe_push (USE_STMT (use_p));
6369 22576 : }
6370 :
6371 45137 : FOR_EACH_VEC_ELT (phis, i, exit_phi)
6372 : {
6373 : /* Replace the uses: */
6374 22561 : orig_name = PHI_RESULT (exit_phi);
6375 :
6376 : /* Look for a single use at the target of the skip edge. */
6377 22561 : if (unify_with_main_loop_p)
6378 : {
6379 30 : use_operand_p use_p;
6380 30 : gimple *user;
6381 30 : if (!single_imm_use (orig_name, &use_p, &user))
6382 0 : gcc_unreachable ();
6383 30 : orig_name = gimple_get_lhs (user);
6384 : }
6385 :
6386 22561 : scalar_result = scalar_results[k];
6387 60879 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
6388 : {
6389 38318 : gphi *use_phi = dyn_cast <gphi *> (use_stmt);
6390 76680 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6391 : {
6392 38340 : if (use_phi
6393 38340 : && (phi_arg_edge_from_use (use_p)->flags & EDGE_ABNORMAL))
6394 : {
6395 0 : gcc_assert (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (orig_name));
6396 0 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (scalar_result) = 1;
6397 : }
6398 38340 : SET_USE (use_p, scalar_result);
6399 : }
6400 38318 : update_stmt (use_stmt);
6401 22561 : }
6402 : }
6403 :
6404 22576 : phis.truncate (0);
6405 : }
6406 22094 : }
6407 :
6408 : /* Return a vector of type VECTYPE that is equal to the vector select
6409 : operation "MASK ? VEC : IDENTITY". Insert the select statements
6410 : before GSI. */
6411 :
6412 : static tree
6413 9 : merge_with_identity (gimple_stmt_iterator *gsi, tree mask, tree vectype,
6414 : tree vec, tree identity)
6415 : {
6416 9 : tree cond = make_temp_ssa_name (vectype, NULL, "cond");
6417 9 : gimple *new_stmt = gimple_build_assign (cond, VEC_COND_EXPR,
6418 : mask, vec, identity);
6419 9 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6420 9 : return cond;
6421 : }
6422 :
6423 : /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
6424 : order, starting with LHS. Insert the extraction statements before GSI and
6425 : associate the new scalar SSA names with variable SCALAR_DEST.
6426 : If MASK is nonzero mask the input and then operate on it unconditionally.
6427 : Return the SSA name for the result. */
6428 :
6429 : static tree
6430 1309 : vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
6431 : tree_code code, tree lhs, tree vector_rhs,
6432 : tree mask)
6433 : {
6434 1309 : tree vectype = TREE_TYPE (vector_rhs);
6435 1309 : tree scalar_type = TREE_TYPE (vectype);
6436 1309 : tree bitsize = TYPE_SIZE (scalar_type);
6437 1309 : unsigned HOST_WIDE_INT vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
6438 1309 : unsigned HOST_WIDE_INT element_bitsize = tree_to_uhwi (bitsize);
6439 :
6440 : /* Re-create a VEC_COND_EXPR to mask the input here in order to be able
6441 : to perform an unconditional element-wise reduction of it. */
6442 1309 : if (mask)
6443 : {
6444 76 : tree masked_vector_rhs = make_temp_ssa_name (vectype, NULL,
6445 : "masked_vector_rhs");
6446 76 : tree neutral_op = neutral_op_for_reduction (scalar_type, code, NULL_TREE,
6447 : false);
6448 76 : tree vector_identity = build_vector_from_val (vectype, neutral_op);
6449 76 : gassign *select = gimple_build_assign (masked_vector_rhs, VEC_COND_EXPR,
6450 : mask, vector_rhs, vector_identity);
6451 76 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6452 76 : vector_rhs = masked_vector_rhs;
6453 : }
6454 :
6455 1309 : for (unsigned HOST_WIDE_INT bit_offset = 0;
6456 5419 : bit_offset < vec_size_in_bits;
6457 4110 : bit_offset += element_bitsize)
6458 : {
6459 4110 : tree bitpos = bitsize_int (bit_offset);
6460 4110 : tree rhs = build3 (BIT_FIELD_REF, scalar_type, vector_rhs,
6461 : bitsize, bitpos);
6462 :
6463 4110 : gassign *stmt = gimple_build_assign (scalar_dest, rhs);
6464 4110 : rhs = make_ssa_name (scalar_dest, stmt);
6465 4110 : gimple_assign_set_lhs (stmt, rhs);
6466 4110 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6467 : /* Fold the vector extract, combining it with a previous reversal
6468 : like seen in PR90579. */
6469 4110 : auto gsi2 = gsi_for_stmt (stmt);
6470 4110 : if (fold_stmt (&gsi2, follow_all_ssa_edges))
6471 450 : update_stmt (gsi_stmt (gsi2));
6472 :
6473 4110 : stmt = gimple_build_assign (scalar_dest, code, lhs, rhs);
6474 4110 : tree new_name = make_ssa_name (scalar_dest, stmt);
6475 4110 : gimple_assign_set_lhs (stmt, new_name);
6476 4110 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6477 4110 : lhs = new_name;
6478 : }
6479 1309 : return lhs;
6480 : }
6481 :
6482 : /* Get a masked internal function equivalent to REDUC_FN. VECTYPE_IN is the
6483 : type of the vector input. */
6484 :
6485 : static internal_fn
6486 3191 : get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
6487 : {
6488 3191 : internal_fn mask_reduc_fn;
6489 3191 : internal_fn mask_len_reduc_fn;
6490 :
6491 3191 : switch (reduc_fn)
6492 : {
6493 0 : case IFN_FOLD_LEFT_PLUS:
6494 0 : mask_reduc_fn = IFN_MASK_FOLD_LEFT_PLUS;
6495 0 : mask_len_reduc_fn = IFN_MASK_LEN_FOLD_LEFT_PLUS;
6496 0 : break;
6497 :
6498 : default:
6499 : return IFN_LAST;
6500 : }
6501 :
6502 0 : if (direct_internal_fn_supported_p (mask_reduc_fn, vectype_in,
6503 : OPTIMIZE_FOR_SPEED))
6504 : return mask_reduc_fn;
6505 0 : if (direct_internal_fn_supported_p (mask_len_reduc_fn, vectype_in,
6506 : OPTIMIZE_FOR_SPEED))
6507 0 : return mask_len_reduc_fn;
6508 : return IFN_LAST;
6509 : }
6510 :
6511 : /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
6512 : statement that sets the live-out value. REDUC_DEF_STMT is the phi
6513 : statement. CODE is the operation performed by STMT_INFO and OPS are
6514 : its scalar operands. REDUC_INDEX is the index of the operand in
6515 : OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
6516 : implements in-order reduction, or IFN_LAST if we should open-code it.
6517 : VECTYPE_IN is the type of the vector input. MASKS specifies the masks
6518 : that should be used to control the operation in a fully-masked loop. */
6519 :
6520 : static bool
6521 977 : vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
6522 : stmt_vec_info stmt_info,
6523 : gimple_stmt_iterator *gsi,
6524 : slp_tree slp_node,
6525 : code_helper code, internal_fn reduc_fn,
6526 : int num_ops, tree vectype_in,
6527 : int reduc_index, vec_loop_masks *masks,
6528 : vec_loop_lens *lens)
6529 : {
6530 977 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
6531 977 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
6532 977 : internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
6533 :
6534 977 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
6535 :
6536 977 : bool is_cond_op = false;
6537 977 : if (!code.is_tree_code ())
6538 : {
6539 29 : code = conditional_internal_fn_code (internal_fn (code));
6540 29 : gcc_assert (code != ERROR_MARK);
6541 : is_cond_op = true;
6542 : }
6543 :
6544 977 : gcc_assert (TREE_CODE_LENGTH (tree_code (code)) == binary_op);
6545 :
6546 977 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out),
6547 : TYPE_VECTOR_SUBPARTS (vectype_in)));
6548 :
6549 : /* ??? We should, when transforming the cycle PHI, record the existing
6550 : scalar def as vector def so looking up the vector def works. This
6551 : would also allow generalizing this for reduction paths of length > 1
6552 : and/or SLP reductions. */
6553 977 : slp_tree reduc_node = SLP_TREE_CHILDREN (slp_node)[reduc_index];
6554 977 : stmt_vec_info reduc_var_def = SLP_TREE_SCALAR_STMTS (reduc_node)[0];
6555 977 : tree reduc_var = gimple_get_lhs (STMT_VINFO_STMT (reduc_var_def));
6556 :
6557 : /* The operands either come from a binary operation or an IFN_COND operation.
6558 : The former is a gimple assign with binary rhs and the latter is a
6559 : gimple call with four arguments. */
6560 977 : gcc_assert (num_ops == 2 || num_ops == 4);
6561 :
6562 977 : auto_vec<tree> vec_oprnds0, vec_opmask;
6563 977 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[(is_cond_op ? 2 : 0)
6564 977 : + (1 - reduc_index)],
6565 : &vec_oprnds0);
6566 : /* For an IFN_COND_OP we also need the vector mask operand. */
6567 977 : if (is_cond_op)
6568 29 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], &vec_opmask);
6569 :
6570 : /* The transform below relies on preserving the original scalar PHI
6571 : and its latch def which we replace. So work backwards from there. */
6572 977 : tree scalar_dest
6573 977 : = gimple_phi_arg_def_from_edge (as_a <gphi *> (STMT_VINFO_STMT
6574 : (reduc_var_def)),
6575 977 : loop_latch_edge (loop));
6576 977 : stmt_vec_info scalar_dest_def_info
6577 977 : = vect_stmt_to_vectorize (loop_vinfo->lookup_def (scalar_dest));
6578 977 : tree scalar_type = TREE_TYPE (scalar_dest);
6579 :
6580 977 : int vec_num = vec_oprnds0.length ();
6581 977 : tree vec_elem_type = TREE_TYPE (vectype_out);
6582 977 : gcc_checking_assert (useless_type_conversion_p (scalar_type, vec_elem_type));
6583 :
6584 977 : tree vector_identity = NULL_TREE;
6585 977 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6586 : {
6587 2 : vector_identity = build_zero_cst (vectype_out);
6588 2 : if (!HONOR_SIGNED_ZEROS (vectype_out))
6589 : ;
6590 : else
6591 : {
6592 2 : gcc_assert (!HONOR_SIGN_DEPENDENT_ROUNDING (vectype_out));
6593 2 : vector_identity = const_unop (NEGATE_EXPR, vectype_out,
6594 : vector_identity);
6595 : }
6596 : }
6597 :
6598 977 : tree scalar_dest_var = vect_create_destination_var (scalar_dest, NULL);
6599 977 : int i;
6600 977 : tree def0;
6601 3263 : FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
6602 : {
6603 1309 : gimple *new_stmt;
6604 1309 : tree mask = NULL_TREE;
6605 1309 : tree len = NULL_TREE;
6606 1309 : tree bias = NULL_TREE;
6607 1309 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6608 : {
6609 9 : tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
6610 : vec_num, vectype_in, i);
6611 9 : if (is_cond_op)
6612 9 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (loop_mask),
6613 9 : loop_mask, vec_opmask[i], gsi);
6614 : else
6615 : mask = loop_mask;
6616 : }
6617 1300 : else if (is_cond_op)
6618 67 : mask = vec_opmask[i];
6619 1309 : if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
6620 : {
6621 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num, vectype_in,
6622 : i, 1, false);
6623 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
6624 0 : bias = build_int_cst (intQI_type_node, biasval);
6625 0 : if (!is_cond_op)
6626 0 : mask = build_minus_one_cst (truth_type_for (vectype_in));
6627 : }
6628 :
6629 : /* Handle MINUS by adding the negative. */
6630 1309 : if (reduc_fn != IFN_LAST && code == MINUS_EXPR)
6631 : {
6632 0 : tree negated = make_ssa_name (vectype_out);
6633 0 : new_stmt = gimple_build_assign (negated, NEGATE_EXPR, def0);
6634 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6635 0 : def0 = negated;
6636 : }
6637 :
6638 9 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
6639 1318 : && mask && mask_reduc_fn == IFN_LAST)
6640 9 : def0 = merge_with_identity (gsi, mask, vectype_out, def0,
6641 : vector_identity);
6642 :
6643 : /* On the first iteration the input is simply the scalar phi
6644 : result, and for subsequent iterations it is the output of
6645 : the preceding operation. */
6646 1309 : if (reduc_fn != IFN_LAST || (mask && mask_reduc_fn != IFN_LAST))
6647 : {
6648 0 : if (mask && len && mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6649 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 5, reduc_var,
6650 : def0, mask, len, bias);
6651 0 : else if (mask && mask_reduc_fn == IFN_MASK_FOLD_LEFT_PLUS)
6652 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 3, reduc_var,
6653 : def0, mask);
6654 : else
6655 0 : new_stmt = gimple_build_call_internal (reduc_fn, 2, reduc_var,
6656 : def0);
6657 : /* For chained SLP reductions the output of the previous reduction
6658 : operation serves as the input of the next. For the final statement
6659 : the output cannot be a temporary - we reuse the original
6660 : scalar destination of the last statement. */
6661 0 : if (i != vec_num - 1)
6662 : {
6663 0 : gimple_set_lhs (new_stmt, scalar_dest_var);
6664 0 : reduc_var = make_ssa_name (scalar_dest_var, new_stmt);
6665 0 : gimple_set_lhs (new_stmt, reduc_var);
6666 : }
6667 : }
6668 : else
6669 : {
6670 1309 : reduc_var = vect_expand_fold_left (gsi, scalar_dest_var,
6671 : tree_code (code), reduc_var, def0,
6672 : mask);
6673 1309 : new_stmt = SSA_NAME_DEF_STMT (reduc_var);
6674 : /* Remove the statement, so that we can use the same code paths
6675 : as for statements that we've just created. */
6676 1309 : gimple_stmt_iterator tmp_gsi = gsi_for_stmt (new_stmt);
6677 1309 : gsi_remove (&tmp_gsi, true);
6678 : }
6679 :
6680 1309 : if (i == vec_num - 1)
6681 : {
6682 977 : gimple_set_lhs (new_stmt, scalar_dest);
6683 977 : vect_finish_replace_stmt (loop_vinfo,
6684 : scalar_dest_def_info,
6685 : new_stmt);
6686 : }
6687 : else
6688 332 : vect_finish_stmt_generation (loop_vinfo,
6689 : scalar_dest_def_info,
6690 : new_stmt, gsi);
6691 :
6692 1309 : slp_node->push_vec_def (new_stmt);
6693 : }
6694 :
6695 977 : return true;
6696 977 : }
6697 :
6698 : /* Function is_nonwrapping_integer_induction.
6699 :
6700 : Check if STMT_VINFO (which is part of loop LOOP) both increments and
6701 : does not cause overflow. */
6702 :
6703 : static bool
6704 414 : is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, class loop *loop)
6705 : {
6706 414 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
6707 414 : tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo);
6708 414 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo);
6709 414 : tree lhs_type = TREE_TYPE (gimple_phi_result (phi));
6710 414 : widest_int ni, max_loop_value, lhs_max;
6711 414 : wi::overflow_type overflow = wi::OVF_NONE;
6712 :
6713 : /* Make sure this is a regular induction. */
6714 414 : if (STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add)
6715 : return false;
6716 :
6717 : /* Make sure the loop is integer based. */
6718 408 : if (TREE_CODE (base) != INTEGER_CST
6719 109 : || TREE_CODE (step) != INTEGER_CST)
6720 : return false;
6721 :
6722 : /* Check that the max size of the loop will not wrap. */
6723 :
6724 109 : if (TYPE_OVERFLOW_UNDEFINED (lhs_type))
6725 : return true;
6726 :
6727 8 : if (! max_stmt_executions (loop, &ni))
6728 : return false;
6729 :
6730 8 : max_loop_value = wi::mul (wi::to_widest (step), ni, TYPE_SIGN (lhs_type),
6731 8 : &overflow);
6732 8 : if (overflow)
6733 : return false;
6734 :
6735 8 : max_loop_value = wi::add (wi::to_widest (base), max_loop_value,
6736 16 : TYPE_SIGN (lhs_type), &overflow);
6737 8 : if (overflow)
6738 : return false;
6739 :
6740 8 : return (wi::min_precision (max_loop_value, TYPE_SIGN (lhs_type))
6741 8 : <= TYPE_PRECISION (lhs_type));
6742 414 : }
6743 :
6744 : /* Check if masking can be supported by inserting a conditional expression.
6745 : CODE is the code for the operation. COND_FN is the conditional internal
6746 : function, if it exists. VECTYPE_IN is the type of the vector input. */
6747 : static bool
6748 6103 : use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
6749 : tree vectype_in)
6750 : {
6751 6103 : if (cond_fn != IFN_LAST
6752 6103 : && direct_internal_fn_supported_p (cond_fn, vectype_in,
6753 : OPTIMIZE_FOR_SPEED))
6754 : return false;
6755 :
6756 4376 : if (code.is_tree_code ())
6757 4356 : switch (tree_code (code))
6758 : {
6759 397 : case DOT_PROD_EXPR:
6760 397 : case SAD_EXPR:
6761 397 : return true;
6762 :
6763 : default:
6764 : break;
6765 : }
6766 : return false;
6767 : }
6768 :
6769 : /* Insert a conditional expression to enable masked vectorization. CODE is the
6770 : code for the operation. VOP is the array of operands. MASK is the loop
6771 : mask. GSI is a statement iterator used to place the new conditional
6772 : expression. */
6773 : static void
6774 4 : build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
6775 : gimple_stmt_iterator *gsi)
6776 : {
6777 4 : switch (tree_code (code))
6778 : {
6779 4 : case DOT_PROD_EXPR:
6780 4 : {
6781 4 : tree vectype = TREE_TYPE (vop[1]);
6782 4 : tree zero = build_zero_cst (vectype);
6783 4 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6784 4 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6785 : mask, vop[1], zero);
6786 4 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6787 4 : vop[1] = masked_op1;
6788 4 : break;
6789 : }
6790 :
6791 0 : case SAD_EXPR:
6792 0 : {
6793 0 : tree vectype = TREE_TYPE (vop[1]);
6794 0 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6795 0 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6796 : mask, vop[1], vop[0]);
6797 0 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6798 0 : vop[1] = masked_op1;
6799 0 : break;
6800 : }
6801 :
6802 0 : default:
6803 0 : gcc_unreachable ();
6804 : }
6805 4 : }
6806 :
6807 : /* Given an operation with CODE in loop reduction path whose reduction PHI is
6808 : specified by REDUC_INFO, the operation has TYPE of scalar result, and its
6809 : input vectype is represented by VECTYPE_IN. The vectype of vectorized result
6810 : may be different from VECTYPE_IN, either in base type or vectype lanes,
6811 : lane-reducing operation is the case. This function check if it is possible,
6812 : and how to perform partial vectorization on the operation in the context
6813 : of LOOP_VINFO. It returns whether partial vectors are possible. The
6814 : caller is responsible to register the usage appropriately. */
6815 :
6816 : static bool
6817 4293 : vect_reduction_update_partial_vector_usage (loop_vec_info loop_vinfo,
6818 : vect_reduc_info reduc_info,
6819 : code_helper code, tree type,
6820 : tree vectype_in)
6821 : {
6822 4293 : enum vect_reduction_type reduc_type = VECT_REDUC_INFO_TYPE (reduc_info);
6823 4293 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
6824 4293 : internal_fn cond_fn
6825 1157 : = ((code.is_internal_fn ()
6826 1157 : && internal_fn_mask_index ((internal_fn)code) != -1)
6827 4293 : ? (internal_fn)code : get_conditional_internal_fn (code, type));
6828 :
6829 4293 : if (reduc_type != FOLD_LEFT_REDUCTION
6830 3411 : && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
6831 7588 : && (cond_fn == IFN_LAST
6832 3295 : || !direct_internal_fn_supported_p (cond_fn, vectype_in,
6833 : OPTIMIZE_FOR_SPEED)))
6834 : {
6835 2079 : if (dump_enabled_p ())
6836 101 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6837 : "can't operate on partial vectors because"
6838 : " no conditional operation is available.\n");
6839 2079 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6840 : }
6841 2214 : else if (reduc_type == FOLD_LEFT_REDUCTION
6842 2214 : && reduc_fn == IFN_LAST
6843 2214 : && !expand_vec_cond_expr_p (vectype_in, truth_type_for (vectype_in)))
6844 : {
6845 0 : if (dump_enabled_p ())
6846 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6847 : "can't operate on partial vectors because"
6848 : " no conditional operation is available.\n");
6849 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6850 : }
6851 2214 : else if (reduc_type == FOLD_LEFT_REDUCTION
6852 882 : && internal_fn_mask_index (reduc_fn) == -1
6853 882 : && FLOAT_TYPE_P (vectype_in)
6854 3096 : && HONOR_SIGN_DEPENDENT_ROUNDING (vectype_in))
6855 : {
6856 0 : if (dump_enabled_p ())
6857 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6858 : "can't operate on partial vectors because"
6859 : " signed zeros cannot be preserved.\n");
6860 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6861 : }
6862 : else
6863 : return true;
6864 : return false;
6865 : }
6866 :
6867 : /* Check if STMT_INFO is a lane-reducing operation that can be vectorized in
6868 : the context of LOOP_VINFO, and vector cost will be recorded in COST_VEC,
6869 : and the analysis is for slp if SLP_NODE is not NULL.
6870 :
6871 : For a lane-reducing operation, the loop reduction path that it lies in,
6872 : may contain normal operation, or other lane-reducing operation of different
6873 : input type size, an example as:
6874 :
6875 : int sum = 0;
6876 : for (i)
6877 : {
6878 : ...
6879 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
6880 : sum += w[i]; // widen-sum <vector(16) char>
6881 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
6882 : sum += n[i]; // normal <vector(4) int>
6883 : ...
6884 : }
6885 :
6886 : Vectorization factor is essentially determined by operation whose input
6887 : vectype has the most lanes ("vector(16) char" in the example), while we
6888 : need to choose input vectype with the least lanes ("vector(4) int" in the
6889 : example) to determine effective number of vector reduction PHIs. */
6890 :
6891 : bool
6892 415227 : vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
6893 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
6894 : {
6895 415227 : gimple *stmt = stmt_info->stmt;
6896 :
6897 415227 : if (!lane_reducing_stmt_p (stmt))
6898 : return false;
6899 :
6900 740 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
6901 :
6902 740 : if (!INTEGRAL_TYPE_P (type) && !SCALAR_FLOAT_TYPE_P (type))
6903 : return false;
6904 :
6905 : /* Do not try to vectorize bit-precision reductions. */
6906 740 : if (!type_has_mode_precision_p (type))
6907 : return false;
6908 :
6909 740 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
6910 :
6911 : /* TODO: Support lane-reducing operation that does not directly participate
6912 : in loop reduction. */
6913 740 : if (!reduc_info)
6914 : return false;
6915 :
6916 : /* Lane-reducing pattern inside any inner loop of LOOP_VINFO is not
6917 : recognized. */
6918 740 : gcc_assert (!nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info));
6919 740 : gcc_assert (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION);
6920 :
6921 2960 : for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
6922 : {
6923 2220 : slp_tree slp_op;
6924 2220 : tree op;
6925 2220 : tree vectype;
6926 2220 : enum vect_def_type dt;
6927 :
6928 2220 : if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
6929 : &slp_op, &dt, &vectype))
6930 : {
6931 0 : if (dump_enabled_p ())
6932 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6933 : "use not simple.\n");
6934 0 : return false;
6935 : }
6936 :
6937 2220 : if (!vectype)
6938 : {
6939 15 : vectype = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op),
6940 : slp_op);
6941 15 : if (!vectype)
6942 : return false;
6943 : }
6944 :
6945 2220 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype))
6946 : {
6947 0 : if (dump_enabled_p ())
6948 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6949 : "incompatible vector types for invariants\n");
6950 : return false;
6951 : }
6952 :
6953 2220 : if (i == STMT_VINFO_REDUC_IDX (stmt_info))
6954 740 : continue;
6955 :
6956 : /* There should be at most one cycle def in the stmt. */
6957 1480 : if (VECTORIZABLE_CYCLE_DEF (dt))
6958 : return false;
6959 : }
6960 :
6961 740 : slp_tree node_in = SLP_TREE_CHILDREN (slp_node)[0];
6962 740 : tree vectype_in = SLP_TREE_VECTYPE (node_in);
6963 740 : gcc_assert (vectype_in);
6964 :
6965 : /* Compute number of effective vector statements for costing from the
6966 : number of input lanes allow for excess lanes in the last input vector. */
6967 740 : unsigned int ncopies_for_cost, excess_elts;
6968 740 : if (!vect_get_num_copies_for_invariant (loop_vinfo, node_in,
6969 : &ncopies_for_cost,
6970 : &excess_elts))
6971 : {
6972 0 : if (dump_enabled_p ())
6973 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6974 : "incompatible vector types for invariants\n");
6975 : return false;
6976 : }
6977 740 : gcc_assert (ncopies_for_cost >= 1);
6978 :
6979 740 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
6980 : {
6981 116 : enum tree_code code = gimple_assign_rhs_code (stmt);
6982 116 : if (vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
6983 116 : code, type, vectype_in))
6984 : {
6985 116 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
6986 116 : internal_fn mask_reduc_fn
6987 116 : = get_masked_reduction_fn (reduc_fn, vectype_in);
6988 116 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
6989 116 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
6990 116 : if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6991 0 : vect_record_loop_len (loop_vinfo, lens, ncopies_for_cost,
6992 : vectype_in, 1);
6993 : else
6994 116 : vect_record_loop_mask (loop_vinfo, masks, ncopies_for_cost,
6995 : vectype_in, NULL);
6996 : }
6997 : }
6998 :
6999 740 : if (vect_is_emulated_mixed_dot_prod (slp_node))
7000 : {
7001 : /* We need extra two invariants: one that contains the minimum signed
7002 : value and one that contains half of its negative. */
7003 15 : int prologue_stmts = 2;
7004 15 : unsigned cost = record_stmt_cost (cost_vec, prologue_stmts,
7005 : scalar_to_vec, slp_node, 0,
7006 : vect_prologue);
7007 15 : if (dump_enabled_p ())
7008 0 : dump_printf (MSG_NOTE, "vectorizable_lane_reducing: "
7009 : "extra prologue_cost = %d .\n", cost);
7010 :
7011 : /* Three dot-products and a subtraction. */
7012 15 : ncopies_for_cost *= 4;
7013 : }
7014 :
7015 740 : record_stmt_cost (cost_vec, (int) ncopies_for_cost, vector_stmt, slp_node,
7016 : 0, vect_body);
7017 :
7018 : /* Transform via vect_transform_reduction. */
7019 740 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
7020 740 : return true;
7021 : }
7022 :
7023 : /* Function vectorizable_reduction.
7024 :
7025 : Check if STMT_INFO performs a reduction operation that can be vectorized.
7026 : If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
7027 : stmt to replace it, put it in VEC_STMT, and insert it at GSI.
7028 : Return true if STMT_INFO is vectorizable in this way.
7029 :
7030 : This function also handles reduction idioms (patterns) that have been
7031 : recognized in advance during vect_pattern_recog. In this case, STMT_INFO
7032 : may be of this form:
7033 : X = pattern_expr (arg0, arg1, ..., X)
7034 : and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
7035 : sequence that had been detected and replaced by the pattern-stmt
7036 : (STMT_INFO).
7037 :
7038 : This function also handles reduction of condition expressions, for example:
7039 : for (int i = 0; i < N; i++)
7040 : if (a[i] < value)
7041 : last = a[i];
7042 : This is handled by vectorising the loop and creating an additional vector
7043 : containing the loop indexes for which "a[i] < value" was true. In the
7044 : function epilogue this is reduced to a single max value and then used to
7045 : index into the vector of results.
7046 :
7047 : In some cases of reduction patterns, the type of the reduction variable X is
7048 : different than the type of the other arguments of STMT_INFO.
7049 : In such cases, the vectype that is used when transforming STMT_INFO into
7050 : a vector stmt is different than the vectype that is used to determine the
7051 : vectorization factor, because it consists of a different number of elements
7052 : than the actual number of elements that are being operated upon in parallel.
7053 :
7054 : For example, consider an accumulation of shorts into an int accumulator.
7055 : On some targets it's possible to vectorize this pattern operating on 8
7056 : shorts at a time (hence, the vectype for purposes of determining the
7057 : vectorization factor should be V8HI); on the other hand, the vectype that
7058 : is used to create the vector form is actually V4SI (the type of the result).
7059 :
7060 : Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
7061 : indicates what is the actual level of parallelism (V8HI in the example), so
7062 : that the right vectorization factor would be derived. This vectype
7063 : corresponds to the type of arguments to the reduction stmt, and should *NOT*
7064 : be used to create the vectorized stmt. The right vectype for the vectorized
7065 : stmt is obtained from the type of the result X:
7066 : get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
7067 :
7068 : This means that, contrary to "regular" reductions (or "regular" stmts in
7069 : general), the following equation:
7070 : STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
7071 : does *NOT* necessarily hold for reduction patterns. */
7072 :
7073 : bool
7074 414487 : vectorizable_reduction (loop_vec_info loop_vinfo,
7075 : stmt_vec_info stmt_info, slp_tree slp_node,
7076 : slp_instance slp_node_instance,
7077 : stmt_vector_for_cost *cost_vec)
7078 : {
7079 414487 : tree vectype_in = NULL_TREE;
7080 414487 : enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
7081 414487 : stmt_vec_info cond_stmt_vinfo = NULL;
7082 414487 : int i;
7083 414487 : int ncopies;
7084 414487 : bool single_defuse_cycle = false;
7085 414487 : tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE;
7086 414487 : tree cond_reduc_val = NULL_TREE;
7087 :
7088 : /* Make sure it was already recognized as a reduction computation. */
7089 414487 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
7090 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def
7091 414487 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
7092 : return false;
7093 :
7094 : /* The reduction meta. */
7095 82718 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
7096 :
7097 82718 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
7098 : {
7099 1392 : gcc_assert (is_a <gphi *> (stmt_info->stmt));
7100 : /* We eventually need to set a vector type on invariant arguments. */
7101 : unsigned j;
7102 : slp_tree child;
7103 4168 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
7104 2784 : if (!vect_maybe_update_slp_op_vectype (child,
7105 : SLP_TREE_VECTYPE (slp_node)))
7106 : {
7107 0 : if (dump_enabled_p ())
7108 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7109 : "incompatible vector types for "
7110 : "invariants\n");
7111 : return false;
7112 : }
7113 2784 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
7114 2784 : && !useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
7115 : SLP_TREE_VECTYPE (child)))
7116 : {
7117 : /* With bools we can have mask and non-mask precision vectors
7118 : or different non-mask precisions. while pattern recog is
7119 : supposed to guarantee consistency here, we do not have
7120 : pattern stmts for PHIs (PR123316).
7121 : Deal with that here instead of ICEing later. */
7122 8 : if (dump_enabled_p ())
7123 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7124 : "incompatible vector type setup from "
7125 : "bool pattern detection\n");
7126 : return false;
7127 : }
7128 : /* Analysis for double-reduction is done on the outer
7129 : loop PHI, nested cycles have no further restrictions. */
7130 1384 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7131 1384 : return true;
7132 : }
7133 :
7134 81326 : if (!is_a <gphi *> (stmt_info->stmt))
7135 : {
7136 8375 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def);
7137 8375 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
7138 8375 : return true;
7139 : }
7140 :
7141 72951 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
7142 72951 : stmt_vec_info phi_info = stmt_info;
7143 72951 : bool double_reduc = false;
7144 72951 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
7145 : {
7146 : /* We arrive here for both the inner loop LC PHI and the
7147 : outer loop PHI. The latter is what we want to analyze the
7148 : reduction with. The LC PHI is handled by vectorizable_lc_phi. */
7149 314 : if (gimple_bb (stmt_info->stmt) != loop->header)
7150 0 : return false;
7151 :
7152 : /* Set loop and phi_info to the inner loop. */
7153 314 : use_operand_p use_p;
7154 314 : gimple *use_stmt;
7155 314 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
7156 : &use_p, &use_stmt);
7157 314 : gcc_assert (res);
7158 314 : phi_info = loop_vinfo->lookup_stmt (use_stmt);
7159 314 : loop = loop->inner;
7160 314 : double_reduc = true;
7161 : }
7162 :
7163 72951 : const bool reduc_chain = reduc_info->is_reduc_chain;
7164 72951 : slp_node_instance->reduc_phis = slp_node;
7165 : /* ??? We're leaving slp_node to point to the PHIs, we only
7166 : need it to get at the number of vector stmts which wasn't
7167 : yet initialized for the instance root. */
7168 :
7169 : /* PHIs should not participate in patterns. */
7170 72951 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7171 72951 : gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
7172 :
7173 : /* Verify following REDUC_IDX from the latch def leads us back to the PHI
7174 : and compute the reduction chain length. Discover the real
7175 : reduction operation stmt on the way (slp_for_stmt_info). */
7176 72951 : unsigned reduc_chain_length = 0;
7177 72951 : stmt_info = NULL;
7178 72951 : slp_tree slp_for_stmt_info = NULL;
7179 72951 : slp_tree vdef_slp = slp_node_instance->root;
7180 160014 : while (vdef_slp != slp_node)
7181 : {
7182 88155 : int reduc_idx = SLP_TREE_REDUC_IDX (vdef_slp);
7183 88155 : if (reduc_idx == -1)
7184 : {
7185 1084 : if (dump_enabled_p ())
7186 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7187 : "reduction chain broken by patterns.\n");
7188 1092 : return false;
7189 : }
7190 87071 : stmt_vec_info vdef = SLP_TREE_REPRESENTATIVE (vdef_slp);
7191 87071 : if (is_a <gphi *> (vdef->stmt))
7192 : {
7193 628 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7194 : /* Do not count PHIs towards the chain length. */
7195 628 : continue;
7196 : }
7197 86443 : gimple_match_op op;
7198 86443 : if (!gimple_extract_op (vdef->stmt, &op))
7199 : {
7200 0 : if (dump_enabled_p ())
7201 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7202 : "reduction chain includes unsupported"
7203 : " statement type.\n");
7204 : return false;
7205 : }
7206 86443 : if (CONVERT_EXPR_CODE_P (op.code))
7207 : {
7208 5392 : if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
7209 : {
7210 8 : if (dump_enabled_p ())
7211 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7212 : "conversion in the reduction chain.\n");
7213 : return false;
7214 : }
7215 5384 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
7216 : }
7217 : else
7218 : {
7219 : /* First non-conversion stmt. */
7220 81051 : if (!slp_for_stmt_info)
7221 71847 : slp_for_stmt_info = vdef_slp;
7222 :
7223 81051 : if (lane_reducing_op_p (op.code))
7224 : {
7225 : /* The last operand of lane-reducing operation is for
7226 : reduction. */
7227 740 : gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
7228 :
7229 740 : slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
7230 740 : tree vectype_op = SLP_TREE_VECTYPE (op_node);
7231 740 : tree type_op = TREE_TYPE (op.ops[0]);
7232 740 : if (!vectype_op)
7233 : {
7234 18 : vectype_op = get_vectype_for_scalar_type (loop_vinfo,
7235 : type_op);
7236 18 : if (!vectype_op
7237 18 : || !vect_maybe_update_slp_op_vectype (op_node,
7238 : vectype_op))
7239 : return false;
7240 : }
7241 :
7242 : /* To accommodate lane-reducing operations of mixed input
7243 : vectypes, choose input vectype with the least lanes for the
7244 : reduction PHI statement, which would result in the most
7245 : ncopies for vectorized reduction results. */
7246 740 : if (!vectype_in
7247 740 : || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
7248 769 : < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
7249 : vectype_in = vectype_op;
7250 : }
7251 80311 : else if (!vectype_in)
7252 71136 : vectype_in = SLP_TREE_VECTYPE (slp_node);
7253 81051 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7254 : }
7255 86435 : reduc_chain_length++;
7256 : }
7257 71859 : if (!slp_for_stmt_info)
7258 : {
7259 12 : if (dump_enabled_p ())
7260 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7261 : "only noop-conversions in the reduction chain.\n");
7262 : return false;
7263 : }
7264 71847 : stmt_info = SLP_TREE_REPRESENTATIVE (slp_for_stmt_info);
7265 :
7266 : /* PHIs should not participate in patterns. */
7267 71847 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7268 :
7269 : /* 1. Is vectorizable reduction? */
7270 : /* Not supportable if the reduction variable is used in the loop, unless
7271 : it's a reduction chain. */
7272 71847 : if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer
7273 0 : && !reduc_chain)
7274 : return false;
7275 :
7276 : /* Reductions that are not used even in an enclosing outer-loop,
7277 : are expected to be "live" (used out of the loop). */
7278 71847 : if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
7279 0 : && !STMT_VINFO_LIVE_P (stmt_info))
7280 : return false;
7281 :
7282 : /* 2. Has this been recognized as a reduction pattern?
7283 :
7284 : Check if STMT represents a pattern that has been recognized
7285 : in earlier analysis stages. For stmts that represent a pattern,
7286 : the STMT_VINFO_RELATED_STMT field records the last stmt in
7287 : the original sequence that constitutes the pattern. */
7288 :
7289 71847 : stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
7290 71847 : if (orig_stmt_info)
7291 : {
7292 5115 : gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
7293 5115 : gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
7294 : }
7295 :
7296 : /* 3. Check the operands of the operation. The first operands are defined
7297 : inside the loop body. The last operand is the reduction variable,
7298 : which is defined by the loop-header-phi. */
7299 :
7300 71847 : tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
7301 71847 : VECT_REDUC_INFO_VECTYPE (reduc_info) = vectype_out;
7302 :
7303 71847 : gimple_match_op op;
7304 71847 : if (!gimple_extract_op (stmt_info->stmt, &op))
7305 0 : gcc_unreachable ();
7306 71847 : bool lane_reducing = lane_reducing_op_p (op.code);
7307 :
7308 71847 : if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
7309 19097 : && !SCALAR_FLOAT_TYPE_P (op.type))
7310 : return false;
7311 :
7312 : /* Do not try to vectorize bit-precision reductions. */
7313 71847 : if (!type_has_mode_precision_p (op.type)
7314 1783 : && op.code != BIT_AND_EXPR
7315 1653 : && op.code != BIT_IOR_EXPR
7316 72339 : && op.code != BIT_XOR_EXPR)
7317 : return false;
7318 :
7319 : /* Lane-reducing ops also never can be used in a SLP reduction group
7320 : since we'll mix lanes belonging to different reductions. But it's
7321 : OK to use them in a reduction chain or when the reduction group
7322 : has just one element. */
7323 71537 : if (lane_reducing
7324 71537 : && !reduc_chain
7325 668 : && SLP_TREE_LANES (slp_node) > 1)
7326 : {
7327 0 : if (dump_enabled_p ())
7328 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7329 : "lane-reducing reduction in reduction group.\n");
7330 : return false;
7331 : }
7332 :
7333 : /* We'll verify the reduction operation only later - avoid
7334 : all operations that mismatch on the number of SLP children. */
7335 143074 : if (op.num_ops != SLP_TREE_CHILDREN (slp_for_stmt_info).length ())
7336 : {
7337 0 : if (dump_enabled_p ())
7338 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7339 : "unsupported reduction operation.\n");
7340 : return false;
7341 : }
7342 :
7343 : /* All uses but the last are expected to be defined in the loop.
7344 : The last use is the reduction variable. In case of nested cycle this
7345 : assumption is not true: we use reduc_index to record the index of the
7346 : reduction variable. */
7347 71537 : slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
7348 71537 : tree *vectype_op = XALLOCAVEC (tree, op.num_ops);
7349 71537 : gcc_assert (op.code != COND_EXPR || !COMPARISON_CLASS_P (op.ops[0]));
7350 229712 : for (i = 0; i < (int) op.num_ops; i++)
7351 : {
7352 : /* The condition of COND_EXPR is checked in vectorizable_condition(). */
7353 158175 : if (i == 0 && op.code == COND_EXPR)
7354 79169 : continue;
7355 :
7356 157302 : stmt_vec_info def_stmt_info;
7357 157302 : enum vect_def_type dt;
7358 157302 : if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
7359 : i, &op.ops[i], &slp_op[i], &dt,
7360 157302 : &vectype_op[i], &def_stmt_info))
7361 : {
7362 0 : if (dump_enabled_p ())
7363 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7364 : "use not simple.\n");
7365 0 : return false;
7366 : }
7367 :
7368 : /* Skip reduction operands, and for an IFN_COND_OP we might hit the
7369 : reduction operand twice (once as definition, once as else). */
7370 157302 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]
7371 314604 : == SLP_TREE_CHILDREN
7372 157302 : (slp_for_stmt_info)[SLP_TREE_REDUC_IDX (slp_for_stmt_info)])
7373 78296 : continue;
7374 :
7375 : /* There should be only one cycle def in the stmt, the one
7376 : leading to reduc_def. */
7377 79006 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]->cycle_info.id != -1)
7378 : return false;
7379 :
7380 79006 : if (!vectype_op[i])
7381 7458 : vectype_op[i]
7382 7458 : = get_vectype_for_scalar_type (loop_vinfo,
7383 7458 : TREE_TYPE (op.ops[i]), slp_op[i]);
7384 :
7385 : /* Record how the non-reduction-def value of COND_EXPR is defined.
7386 : ??? For a chain of multiple CONDs we'd have to match them up all. */
7387 79006 : if (op.code == COND_EXPR && reduc_chain_length == 1)
7388 : {
7389 850 : if (dt == vect_constant_def)
7390 : {
7391 124 : cond_reduc_dt = dt;
7392 124 : cond_reduc_val = op.ops[i];
7393 : }
7394 726 : else if (dt == vect_induction_def
7395 414 : && def_stmt_info
7396 1140 : && is_nonwrapping_integer_induction (def_stmt_info, loop))
7397 : {
7398 109 : cond_reduc_dt = dt;
7399 109 : cond_stmt_vinfo = def_stmt_info;
7400 : }
7401 : }
7402 : }
7403 :
7404 71537 : enum vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7405 : /* If we have a condition reduction, see if we can simplify it further. */
7406 71537 : if (reduction_type == COND_REDUCTION)
7407 : {
7408 861 : if (SLP_TREE_LANES (slp_node) != 1)
7409 : return false;
7410 :
7411 : /* When the condition uses the reduction value in the condition, fail. */
7412 837 : if (SLP_TREE_REDUC_IDX (slp_node) == 0)
7413 : {
7414 0 : if (dump_enabled_p ())
7415 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7416 : "condition depends on previous iteration\n");
7417 : return false;
7418 : }
7419 :
7420 837 : if (reduc_chain_length == 1
7421 837 : && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in,
7422 : OPTIMIZE_FOR_SPEED)
7423 814 : || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
7424 : vectype_in,
7425 : OPTIMIZE_FOR_SPEED)))
7426 : {
7427 0 : if (dump_enabled_p ())
7428 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7429 : "optimizing condition reduction with"
7430 : " FOLD_EXTRACT_LAST.\n");
7431 0 : VECT_REDUC_INFO_TYPE (reduc_info) = EXTRACT_LAST_REDUCTION;
7432 : }
7433 837 : else if (cond_reduc_dt == vect_induction_def)
7434 : {
7435 109 : tree base
7436 : = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo);
7437 109 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo);
7438 :
7439 109 : gcc_assert (TREE_CODE (base) == INTEGER_CST
7440 : && TREE_CODE (step) == INTEGER_CST);
7441 109 : cond_reduc_val = NULL_TREE;
7442 109 : enum tree_code cond_reduc_op_code = ERROR_MARK;
7443 109 : tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
7444 109 : if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
7445 : ;
7446 : /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
7447 : above base; punt if base is the minimum value of the type for
7448 : MAX_EXPR or maximum value of the type for MIN_EXPR for now. */
7449 97 : else if (tree_int_cst_sgn (step) == -1)
7450 : {
7451 18 : cond_reduc_op_code = MIN_EXPR;
7452 18 : if (tree_int_cst_sgn (base) == -1)
7453 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7454 18 : else if (tree_int_cst_lt (base,
7455 18 : TYPE_MAX_VALUE (TREE_TYPE (base))))
7456 18 : cond_reduc_val
7457 18 : = int_const_binop (PLUS_EXPR, base, integer_one_node);
7458 : }
7459 : else
7460 : {
7461 79 : cond_reduc_op_code = MAX_EXPR;
7462 79 : if (tree_int_cst_sgn (base) == 1)
7463 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7464 79 : else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base)),
7465 : base))
7466 79 : cond_reduc_val
7467 79 : = int_const_binop (MINUS_EXPR, base, integer_one_node);
7468 : }
7469 97 : if (cond_reduc_val)
7470 : {
7471 97 : if (dump_enabled_p ())
7472 61 : dump_printf_loc (MSG_NOTE, vect_location,
7473 : "condition expression based on "
7474 : "integer induction.\n");
7475 97 : VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_op_code;
7476 97 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info)
7477 97 : = cond_reduc_val;
7478 97 : VECT_REDUC_INFO_TYPE (reduc_info) = INTEGER_INDUC_COND_REDUCTION;
7479 : }
7480 : }
7481 728 : else if (cond_reduc_dt == vect_constant_def)
7482 : {
7483 114 : enum vect_def_type cond_initial_dt;
7484 114 : tree cond_initial_val = vect_phi_initial_value (reduc_def_phi);
7485 114 : vect_is_simple_use (cond_initial_val, loop_vinfo, &cond_initial_dt);
7486 114 : if (cond_initial_dt == vect_constant_def
7487 145 : && types_compatible_p (TREE_TYPE (cond_initial_val),
7488 31 : TREE_TYPE (cond_reduc_val)))
7489 : {
7490 31 : tree lt, gt;
7491 31 : enum tree_code cond_reduc_code = ERROR_MARK;
7492 31 : if ((lt = fold_binary (LT_EXPR, boolean_type_node,
7493 : cond_initial_val, cond_reduc_val))
7494 31 : && integer_onep (lt))
7495 : cond_reduc_code = MAX_EXPR;
7496 13 : else if ((gt = fold_binary (GT_EXPR, boolean_type_node,
7497 : cond_initial_val, cond_reduc_val))
7498 13 : && integer_onep (gt))
7499 : cond_reduc_code = MIN_EXPR;
7500 : if (cond_reduc_code != ERROR_MARK)
7501 : {
7502 25 : if (dump_enabled_p ())
7503 16 : dump_printf_loc (MSG_NOTE, vect_location,
7504 : "condition expression based on "
7505 : "compile time constant.\n");
7506 : /* Record reduction code at analysis stage. */
7507 25 : VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_code;
7508 25 : VECT_REDUC_INFO_TYPE (reduc_info) = CONST_COND_REDUCTION;
7509 : }
7510 : }
7511 : }
7512 : }
7513 :
7514 71513 : if (STMT_VINFO_LIVE_P (phi_info))
7515 : return false;
7516 :
7517 71513 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
7518 :
7519 71513 : gcc_assert (ncopies >= 1);
7520 :
7521 71513 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
7522 :
7523 : /* 4.2. Check support for the epilog operation.
7524 :
7525 : If STMT represents a reduction pattern, then the type of the
7526 : reduction variable may be different than the type of the rest
7527 : of the arguments. For example, consider the case of accumulation
7528 : of shorts into an int accumulator; The original code:
7529 : S1: int_a = (int) short_a;
7530 : orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
7531 :
7532 : was replaced with:
7533 : STMT: int_acc = widen_sum <short_a, int_acc>
7534 :
7535 : This means that:
7536 : 1. The tree-code that is used to create the vector operation in the
7537 : epilog code (that reduces the partial results) is not the
7538 : tree-code of STMT, but is rather the tree-code of the original
7539 : stmt from the pattern that STMT is replacing. I.e, in the example
7540 : above we want to use 'widen_sum' in the loop, but 'plus' in the
7541 : epilog.
7542 : 2. The type (mode) we use to check available target support
7543 : for the vector operation to be created in the *epilog*, is
7544 : determined by the type of the reduction variable (in the example
7545 : above we'd check this: optab_handler (plus_optab, vect_int_mode])).
7546 : However the type (mode) we use to check available target support
7547 : for the vector operation to be created *inside the loop*, is
7548 : determined by the type of the other arguments to STMT (in the
7549 : example we'd check this: optab_handler (widen_sum_optab,
7550 : vect_short_mode)).
7551 :
7552 : This is contrary to "regular" reductions, in which the types of all
7553 : the arguments are the same as the type of the reduction variable.
7554 : For "regular" reductions we can therefore use the same vector type
7555 : (and also the same tree-code) when generating the epilog code and
7556 : when generating the code inside the loop. */
7557 :
7558 71513 : code_helper orig_code = VECT_REDUC_INFO_CODE (reduc_info);
7559 :
7560 : /* If conversion might have created a conditional operation like
7561 : IFN_COND_ADD already. Use the internal code for the following checks. */
7562 71513 : if (orig_code.is_internal_fn ())
7563 : {
7564 6835 : tree_code new_code = conditional_internal_fn_code (internal_fn (orig_code));
7565 6835 : orig_code = new_code != ERROR_MARK ? new_code : orig_code;
7566 : }
7567 :
7568 71513 : VECT_REDUC_INFO_CODE (reduc_info) = orig_code;
7569 :
7570 71513 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7571 71513 : if (reduction_type == TREE_CODE_REDUCTION)
7572 : {
7573 : /* Check whether it's ok to change the order of the computation.
7574 : Generally, when vectorizing a reduction we change the order of the
7575 : computation. This may change the behavior of the program in some
7576 : cases, so we need to check that this is ok. One exception is when
7577 : vectorizing an outer-loop: the inner-loop is executed sequentially,
7578 : and therefore vectorizing reductions in the inner-loop during
7579 : outer-loop vectorization is safe. Likewise when we are vectorizing
7580 : a series of reductions using SLP and the VF is one the reductions
7581 : are performed in scalar order. */
7582 70676 : if (!reduc_chain
7583 70676 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
7584 : ;
7585 70504 : else if (needs_fold_left_reduction_p (op.type, orig_code))
7586 : {
7587 : /* When vectorizing a reduction chain w/o SLP the reduction PHI
7588 : is not directly used in stmt. */
7589 5654 : if (reduc_chain_length != 1)
7590 : {
7591 105 : if (dump_enabled_p ())
7592 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7593 : "in-order reduction chain without SLP.\n");
7594 : return false;
7595 : }
7596 : /* Code generation doesn't support function calls other
7597 : than .COND_*. */
7598 5549 : if (!op.code.is_tree_code ()
7599 5737 : && !(op.code.is_internal_fn ()
7600 94 : && conditional_internal_fn_code (internal_fn (op.code))
7601 : != ERROR_MARK))
7602 : {
7603 18 : if (dump_enabled_p ())
7604 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7605 : "in-order reduction chain operation not "
7606 : "supported.\n");
7607 : return false;
7608 : }
7609 5531 : VECT_REDUC_INFO_TYPE (reduc_info)
7610 5531 : = reduction_type = FOLD_LEFT_REDUCTION;
7611 : }
7612 64850 : else if (!commutative_binary_op_p (orig_code, op.type)
7613 64850 : || !associative_binary_op_p (orig_code, op.type))
7614 : {
7615 180 : if (dump_enabled_p ())
7616 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7617 : "reduction: not commutative/associative\n");
7618 : return false;
7619 : }
7620 : }
7621 :
7622 5531 : if ((reduction_type == COND_REDUCTION
7623 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7624 : || reduction_type == CONST_COND_REDUCTION
7625 65679 : || reduction_type == EXTRACT_LAST_REDUCTION)
7626 837 : && ncopies > 1)
7627 : {
7628 276 : if (dump_enabled_p ())
7629 60 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7630 : "multiple types in condition reduction.\n");
7631 : return false;
7632 : }
7633 :
7634 : /* See if we can convert a mask vector to a corresponding bool data vector
7635 : to perform the epilogue reduction. */
7636 70934 : tree alt_vectype_out = NULL_TREE;
7637 70934 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out))
7638 : {
7639 1179 : alt_vectype_out
7640 2358 : = get_related_vectype_for_scalar_type (loop_vinfo->vector_mode,
7641 1179 : TREE_TYPE (vectype_out),
7642 : TYPE_VECTOR_SUBPARTS
7643 : (vectype_out));
7644 1179 : if (!alt_vectype_out
7645 1179 : || maybe_ne (TYPE_VECTOR_SUBPARTS (alt_vectype_out),
7646 2358 : TYPE_VECTOR_SUBPARTS (vectype_out))
7647 2358 : || !expand_vec_cond_expr_p (alt_vectype_out, vectype_out))
7648 : alt_vectype_out = NULL_TREE;
7649 : }
7650 :
7651 70934 : internal_fn reduc_fn = IFN_LAST;
7652 70934 : if (reduction_type == TREE_CODE_REDUCTION
7653 70934 : || reduction_type == FOLD_LEFT_REDUCTION
7654 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7655 561 : || reduction_type == CONST_COND_REDUCTION)
7656 : {
7657 64956 : if (reduction_type == FOLD_LEFT_REDUCTION
7658 75225 : ? fold_left_reduction_fn (orig_code, &reduc_fn)
7659 64956 : : reduction_fn_for_scalar_code (orig_code, &reduc_fn))
7660 : {
7661 69808 : internal_fn sbool_fn = IFN_LAST;
7662 69808 : if (reduc_fn == IFN_LAST)
7663 : ;
7664 67714 : else if ((!VECTOR_BOOLEAN_TYPE_P (vectype_out)
7665 1179 : || (GET_MODE_CLASS (TYPE_MODE (vectype_out))
7666 : == MODE_VECTOR_BOOL))
7667 134249 : && direct_internal_fn_supported_p (reduc_fn, vectype_out,
7668 : OPTIMIZE_FOR_SPEED))
7669 : ;
7670 18636 : else if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
7671 1179 : && sbool_reduction_fn_for_fn (reduc_fn, &sbool_fn)
7672 19815 : && direct_internal_fn_supported_p (sbool_fn, vectype_out,
7673 : OPTIMIZE_FOR_SPEED))
7674 131 : reduc_fn = sbool_fn;
7675 18505 : else if (reduction_type != FOLD_LEFT_REDUCTION
7676 18505 : && alt_vectype_out
7677 18505 : && direct_internal_fn_supported_p (reduc_fn, alt_vectype_out,
7678 : OPTIMIZE_FOR_SPEED))
7679 805 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7680 : else
7681 : {
7682 17700 : if (dump_enabled_p ())
7683 966 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7684 : "reduc op not supported by target.\n");
7685 :
7686 17700 : reduc_fn = IFN_LAST;
7687 : }
7688 : }
7689 : else
7690 : {
7691 679 : if (dump_enabled_p ())
7692 48 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7693 : "no reduc code for scalar code.\n");
7694 :
7695 : return false;
7696 : }
7697 69808 : if (reduc_fn == IFN_LAST
7698 69808 : && VECTOR_BOOLEAN_TYPE_P (vectype_out))
7699 : {
7700 243 : if (!alt_vectype_out)
7701 : {
7702 12 : if (dump_enabled_p ())
7703 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7704 : "cannot turn mask into bool data vector for "
7705 : "reduction epilogue.\n");
7706 : return false;
7707 : }
7708 231 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7709 : }
7710 : }
7711 447 : else if (reduction_type == COND_REDUCTION)
7712 : {
7713 447 : int scalar_precision
7714 447 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
7715 447 : cr_index_scalar_type = make_unsigned_type (scalar_precision);
7716 447 : cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
7717 : vectype_out);
7718 :
7719 447 : if (direct_internal_fn_supported_p (IFN_REDUC_MAX, cr_index_vector_type,
7720 : OPTIMIZE_FOR_SPEED))
7721 22 : reduc_fn = IFN_REDUC_MAX;
7722 : }
7723 70243 : VECT_REDUC_INFO_FN (reduc_info) = reduc_fn;
7724 :
7725 70243 : if (reduction_type != EXTRACT_LAST_REDUCTION
7726 : && reduc_fn == IFN_LAST
7727 : && !nunits_out.is_constant ())
7728 : {
7729 : if (dump_enabled_p ())
7730 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7731 : "missing target support for reduction on"
7732 : " variable-length vectors.\n");
7733 : return false;
7734 : }
7735 :
7736 : /* For SLP reductions, see if there is a neutral value we can use. */
7737 70243 : tree neutral_op = NULL_TREE;
7738 70243 : tree initial_value = NULL_TREE;
7739 70243 : if (reduc_chain)
7740 522 : initial_value = vect_phi_initial_value (reduc_def_phi);
7741 70243 : neutral_op = neutral_op_for_reduction (TREE_TYPE
7742 : (gimple_phi_result (reduc_def_phi)),
7743 : orig_code, initial_value);
7744 70243 : VECT_REDUC_INFO_NEUTRAL_OP (reduc_info) = neutral_op;
7745 :
7746 70243 : if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
7747 : {
7748 : /* We can't support in-order reductions of code such as this:
7749 :
7750 : for (int i = 0; i < n1; ++i)
7751 : for (int j = 0; j < n2; ++j)
7752 : l += a[j];
7753 :
7754 : since GCC effectively transforms the loop when vectorizing:
7755 :
7756 : for (int i = 0; i < n1 / VF; ++i)
7757 : for (int j = 0; j < n2; ++j)
7758 : for (int k = 0; k < VF; ++k)
7759 : l += a[j];
7760 :
7761 : which is a reassociation of the original operation. */
7762 66 : if (dump_enabled_p ())
7763 20 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7764 : "in-order double reduction not supported.\n");
7765 :
7766 : return false;
7767 : }
7768 :
7769 70177 : if (reduction_type == FOLD_LEFT_REDUCTION
7770 4786 : && SLP_TREE_LANES (slp_node) > 1
7771 194 : && !reduc_chain)
7772 : {
7773 : /* We cannot use in-order reductions in this case because there is
7774 : an implicit reassociation of the operations involved. */
7775 72 : if (dump_enabled_p ())
7776 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7777 : "in-order unchained SLP reductions not supported.\n");
7778 : return false;
7779 : }
7780 :
7781 : /* For double reductions, and for SLP reductions with a neutral value,
7782 : we construct a variable-length initial vector by loading a vector
7783 : full of the neutral value and then shift-and-inserting the start
7784 : values into the low-numbered elements. This is however not needed
7785 : when neutral and initial value are equal or we can handle the
7786 : initial value via adjustment in the epilogue. */
7787 70105 : if ((double_reduc || neutral_op)
7788 : && !nunits_out.is_constant ()
7789 : && reduction_type != INTEGER_INDUC_COND_REDUCTION
7790 : && !((SLP_TREE_LANES (slp_node) == 1 || reduc_chain)
7791 : && neutral_op
7792 : && (!double_reduc
7793 : || operand_equal_p (neutral_op,
7794 : vect_phi_initial_value (reduc_def_phi))))
7795 : && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT,
7796 : vectype_out, OPTIMIZE_FOR_BOTH))
7797 : {
7798 : if (dump_enabled_p ())
7799 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7800 : "reduction on variable-length vectors requires"
7801 : " target support for a vector-shift-and-insert"
7802 : " operation.\n");
7803 : return false;
7804 : }
7805 :
7806 : /* Check extra constraints for variable-length unchained SLP reductions. */
7807 70105 : if (!reduc_chain
7808 : && !nunits_out.is_constant ())
7809 : {
7810 : /* We checked above that we could build the initial vector when
7811 : there's a neutral element value. Check here for the case in
7812 : which each SLP statement has its own initial value and in which
7813 : that value needs to be repeated for every instance of the
7814 : statement within the initial vector. */
7815 : unsigned int group_size = SLP_TREE_LANES (slp_node);
7816 : if (!neutral_op
7817 : && !can_duplicate_and_interleave_p (loop_vinfo, group_size,
7818 : TREE_TYPE (vectype_out)))
7819 : {
7820 : if (dump_enabled_p ())
7821 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7822 : "unsupported form of SLP reduction for"
7823 : " variable-length vectors: cannot build"
7824 : " initial vector.\n");
7825 : return false;
7826 : }
7827 : /* The epilogue code relies on the number of elements being a multiple
7828 : of the group size. The duplicate-and-interleave approach to setting
7829 : up the initial vector does too. */
7830 : if (!multiple_p (nunits_out, group_size))
7831 : {
7832 : if (dump_enabled_p ())
7833 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7834 : "unsupported form of SLP reduction for"
7835 : " variable-length vectors: the vector size"
7836 : " is not a multiple of the number of results.\n");
7837 : return false;
7838 : }
7839 : }
7840 :
7841 70105 : if (reduction_type == COND_REDUCTION)
7842 : {
7843 447 : widest_int ni;
7844 :
7845 447 : if (! max_loop_iterations (loop, &ni))
7846 : {
7847 8 : if (dump_enabled_p ())
7848 0 : dump_printf_loc (MSG_NOTE, vect_location,
7849 : "loop count not known, cannot create cond "
7850 : "reduction.\n");
7851 : return false;
7852 : }
7853 : /* Convert backedges to iterations. */
7854 439 : ni += 1;
7855 :
7856 : /* The additional index will be the same type as the condition. Check
7857 : that the loop can fit into this less one (because we'll use up the
7858 : zero slot for when there are no matches). */
7859 439 : tree max_index = TYPE_MAX_VALUE (cr_index_scalar_type);
7860 439 : if (wi::geu_p (ni, wi::to_widest (max_index)))
7861 : {
7862 90 : if (dump_enabled_p ())
7863 54 : dump_printf_loc (MSG_NOTE, vect_location,
7864 : "loop size is greater than data size.\n");
7865 : return false;
7866 : }
7867 447 : }
7868 :
7869 : /* In case the vectorization factor (VF) is bigger than the number
7870 : of elements that we can fit in a vectype (nunits), we have to generate
7871 : more than one vector stmt - i.e - we need to "unroll" the
7872 : vector stmt by a factor VF/nunits. For more details see documentation
7873 : in vectorizable_operation. */
7874 :
7875 : /* If the reduction is used in an outer loop we need to generate
7876 : VF intermediate results, like so (e.g. for ncopies=2):
7877 : r0 = phi (init, r0)
7878 : r1 = phi (init, r1)
7879 : r0 = x0 + r0;
7880 : r1 = x1 + r1;
7881 : (i.e. we generate VF results in 2 registers).
7882 : In this case we have a separate def-use cycle for each copy, and therefore
7883 : for each copy we get the vector def for the reduction variable from the
7884 : respective phi node created for this copy.
7885 :
7886 : Otherwise (the reduction is unused in the loop nest), we can combine
7887 : together intermediate results, like so (e.g. for ncopies=2):
7888 : r = phi (init, r)
7889 : r = x0 + r;
7890 : r = x1 + r;
7891 : (i.e. we generate VF/2 results in a single register).
7892 : In this case for each copy we get the vector def for the reduction variable
7893 : from the vectorized reduction operation generated in the previous iteration.
7894 :
7895 : This only works when we see both the reduction PHI and its only consumer
7896 : in vectorizable_reduction and there are no intermediate stmts
7897 : participating. When unrolling we want each unrolled iteration to have its
7898 : own reduction accumulator since one of the main goals of unrolling a
7899 : reduction is to reduce the aggregate loop-carried latency. */
7900 70007 : if (ncopies > 1
7901 70007 : && !reduc_chain
7902 8156 : && SLP_TREE_LANES (slp_node) == 1
7903 7983 : && (STMT_VINFO_RELEVANT (stmt_info) <= vect_used_only_live)
7904 7960 : && reduc_chain_length == 1
7905 7558 : && loop_vinfo->suggested_unroll_factor == 1)
7906 70007 : single_defuse_cycle = true;
7907 :
7908 70007 : if (single_defuse_cycle && !lane_reducing)
7909 : {
7910 6601 : gcc_assert (op.code != COND_EXPR);
7911 :
7912 : /* 4. check support for the operation in the loop
7913 :
7914 : This isn't necessary for the lane reduction codes, since they
7915 : can only be produced by pattern matching, and it's up to the
7916 : pattern matcher to test for support. The main reason for
7917 : specifically skipping this step is to avoid rechecking whether
7918 : mixed-sign dot-products can be implemented using signed
7919 : dot-products. */
7920 6601 : machine_mode vec_mode = TYPE_MODE (vectype_in);
7921 6601 : if (!directly_supported_p (op.code, vectype_in, optab_vector))
7922 : {
7923 2076 : if (dump_enabled_p ())
7924 36 : dump_printf (MSG_NOTE, "op not supported by target.\n");
7925 4152 : if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
7926 2076 : || !vect_can_vectorize_without_simd_p (op.code))
7927 : single_defuse_cycle = false;
7928 : else
7929 10 : if (dump_enabled_p ())
7930 0 : dump_printf (MSG_NOTE, "proceeding using word mode.\n");
7931 : }
7932 :
7933 6601 : if (vect_emulated_vector_p (vectype_in)
7934 6601 : && !vect_can_vectorize_without_simd_p (op.code))
7935 : {
7936 0 : if (dump_enabled_p ())
7937 0 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
7938 : return false;
7939 : }
7940 : }
7941 70007 : if (dump_enabled_p () && single_defuse_cycle)
7942 731 : dump_printf_loc (MSG_NOTE, vect_location,
7943 : "using single def-use cycle for reduction by reducing "
7944 : "multiple vectors to one in the loop body\n");
7945 70007 : VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info) = single_defuse_cycle;
7946 :
7947 : /* For lane-reducing operation, the below processing related to single
7948 : defuse-cycle will be done in its own vectorizable function. One more
7949 : thing to note is that the operation must not be involved in fold-left
7950 : reduction. */
7951 70007 : single_defuse_cycle &= !lane_reducing;
7952 :
7953 70007 : if (single_defuse_cycle || reduction_type == FOLD_LEFT_REDUCTION)
7954 29842 : for (i = 0; i < (int) op.num_ops; i++)
7955 20688 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_op[i]))
7956 : {
7957 0 : if (dump_enabled_p ())
7958 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7959 : "incompatible vector types for invariants\n");
7960 : return false;
7961 : }
7962 :
7963 70007 : vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
7964 : reduction_type, ncopies, cost_vec);
7965 : /* Cost the reduction op inside the loop if transformed via
7966 : vect_transform_reduction for non-lane-reducing operation. Otherwise
7967 : this is costed by the separate vectorizable_* routines. */
7968 70007 : if (single_defuse_cycle)
7969 4535 : record_stmt_cost (cost_vec, ncopies, vector_stmt,
7970 : slp_for_stmt_info, 0, vect_body);
7971 :
7972 70007 : if (dump_enabled_p ()
7973 70007 : && reduction_type == FOLD_LEFT_REDUCTION)
7974 264 : dump_printf_loc (MSG_NOTE, vect_location,
7975 : "using an in-order (fold-left) reduction.\n");
7976 70007 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7977 :
7978 : /* All but single defuse-cycle optimized and fold-left reductions go
7979 : through their own vectorizable_* routines. */
7980 70007 : stmt_vec_info tem
7981 70007 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (slp_node_instance));
7982 70007 : if (!single_defuse_cycle && reduction_type != FOLD_LEFT_REDUCTION)
7983 60853 : STMT_VINFO_DEF_TYPE (tem) = vect_internal_def;
7984 : else
7985 : {
7986 9154 : STMT_VINFO_DEF_TYPE (tem) = vect_reduction_def;
7987 9154 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
7988 9154 : && vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
7989 : op.code, op.type,
7990 : vectype_in))
7991 : {
7992 2098 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
7993 2098 : internal_fn mask_reduc_fn
7994 2098 : = get_masked_reduction_fn (reduc_fn, vectype_in);
7995 2098 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
7996 2098 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
7997 2098 : if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
7998 0 : vect_record_loop_len (loop_vinfo, lens, ncopies, vectype_in, 1);
7999 : else
8000 2098 : vect_record_loop_mask (loop_vinfo, masks, ncopies,
8001 : vectype_in, NULL);
8002 : }
8003 : }
8004 : return true;
8005 : }
8006 :
8007 : /* STMT_INFO is a dot-product reduction whose multiplication operands
8008 : have different signs. Emit a sequence to emulate the operation
8009 : using a series of signed DOT_PROD_EXPRs and return the last
8010 : statement generated. VEC_DEST is the result of the vector operation
8011 : and VOP lists its inputs. */
8012 :
8013 : static gassign *
8014 4 : vect_emulate_mixed_dot_prod (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
8015 : gimple_stmt_iterator *gsi, tree vec_dest,
8016 : tree vop[3])
8017 : {
8018 4 : tree wide_vectype = signed_type_for (TREE_TYPE (vec_dest));
8019 4 : tree narrow_vectype = signed_type_for (TREE_TYPE (vop[0]));
8020 4 : tree narrow_elttype = TREE_TYPE (narrow_vectype);
8021 4 : gimple *new_stmt;
8022 :
8023 : /* Make VOP[0] the unsigned operand VOP[1] the signed operand. */
8024 4 : if (!TYPE_UNSIGNED (TREE_TYPE (vop[0])))
8025 0 : std::swap (vop[0], vop[1]);
8026 :
8027 : /* Convert all inputs to signed types. */
8028 12 : for (int i = 1; i < 3; ++i)
8029 8 : if (TYPE_UNSIGNED (TREE_TYPE (vop[i])))
8030 : {
8031 0 : tree tmp = make_ssa_name (signed_type_for (TREE_TYPE (vop[i])));
8032 0 : new_stmt = gimple_build_assign (tmp, NOP_EXPR, vop[i]);
8033 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8034 0 : vop[i] = tmp;
8035 : }
8036 :
8037 : /* In the comments below we assume 8-bit inputs for simplicity,
8038 : but the approach works for any full integer type. */
8039 :
8040 : /* Create a vector of -128. */
8041 4 : tree min_narrow_elttype = TYPE_MIN_VALUE (narrow_elttype);
8042 4 : tree min_narrow = build_vector_from_val (TREE_TYPE (vop[0]),
8043 4 : fold_convert
8044 : (TREE_TYPE (TREE_TYPE (vop[0])),
8045 : min_narrow_elttype));
8046 :
8047 : /* Create a vector of 64. */
8048 4 : auto half_wi = wi::lrshift (wi::to_wide (min_narrow_elttype), 1);
8049 4 : tree half_narrow = wide_int_to_tree (narrow_elttype, half_wi);
8050 4 : half_narrow = build_vector_from_val (narrow_vectype, half_narrow);
8051 :
8052 : /* Emit: SUB_RES = VOP[0] - 128 in an unsigned type. */
8053 4 : tree sub_res = make_ssa_name (TREE_TYPE (vop[0]));
8054 4 : new_stmt = gimple_build_assign (sub_res, PLUS_EXPR, vop[0], min_narrow);
8055 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8056 :
8057 4 : vop[0] = make_ssa_name (narrow_vectype);
8058 4 : new_stmt = gimple_build_assign (vop[0], VIEW_CONVERT_EXPR,
8059 : build1 (VIEW_CONVERT_EXPR, narrow_vectype,
8060 : sub_res));
8061 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8062 :
8063 : /* Emit:
8064 :
8065 : STAGE1 = DOT_PROD_EXPR <VOP[1], 64, VOP[2]>;
8066 : STAGE2 = DOT_PROD_EXPR <VOP[1], 64, STAGE1>;
8067 : STAGE3 = DOT_PROD_EXPR <SUB_RES, -128, STAGE2>;
8068 :
8069 : on the basis that x * y == (x - 128) * y + 64 * y + 64 * y
8070 : Doing the two 64 * y steps first allows more time to compute x. */
8071 4 : tree stage1 = make_ssa_name (wide_vectype);
8072 4 : new_stmt = gimple_build_assign (stage1, DOT_PROD_EXPR,
8073 : vop[1], half_narrow, vop[2]);
8074 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8075 :
8076 4 : tree stage2 = make_ssa_name (wide_vectype);
8077 4 : new_stmt = gimple_build_assign (stage2, DOT_PROD_EXPR,
8078 : vop[1], half_narrow, stage1);
8079 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8080 :
8081 4 : tree stage3 = make_ssa_name (wide_vectype);
8082 4 : new_stmt = gimple_build_assign (stage3, DOT_PROD_EXPR,
8083 : vop[0], vop[1], stage2);
8084 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8085 :
8086 : /* Convert STAGE3 to the reduction type. */
8087 4 : return gimple_build_assign (vec_dest, CONVERT_EXPR, stage3);
8088 4 : }
8089 :
8090 : /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
8091 : value. */
8092 :
8093 : bool
8094 2692 : vect_transform_reduction (loop_vec_info loop_vinfo,
8095 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8096 : slp_tree slp_node)
8097 : {
8098 2692 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
8099 2692 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
8100 :
8101 2692 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
8102 :
8103 2692 : if (nested_in_vect_loop_p (loop, stmt_info))
8104 : {
8105 0 : loop = loop->inner;
8106 0 : gcc_assert (VECT_REDUC_INFO_DEF_TYPE (reduc_info)
8107 : == vect_double_reduction_def);
8108 : }
8109 :
8110 2692 : gimple_match_op op;
8111 2692 : if (!gimple_extract_op (stmt_info->stmt, &op))
8112 0 : gcc_unreachable ();
8113 :
8114 : /* All uses but the last are expected to be defined in the loop.
8115 : The last use is the reduction variable. In case of nested cycle this
8116 : assumption is not true: we use reduc_index to record the index of the
8117 : reduction variable. */
8118 2692 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
8119 2692 : tree vectype_in = SLP_TREE_VECTYPE (slp_node);
8120 2692 : if (lane_reducing_op_p (op.code))
8121 281 : vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]);
8122 :
8123 2692 : code_helper code = canonicalize_code (op.code, op.type);
8124 2692 : internal_fn cond_fn
8125 484 : = ((code.is_internal_fn ()
8126 484 : && internal_fn_mask_index ((internal_fn)code) != -1)
8127 2692 : ? (internal_fn)code : get_conditional_internal_fn (code, op.type));
8128 :
8129 2692 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
8130 2692 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
8131 2692 : bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
8132 :
8133 : /* Transform. */
8134 2692 : tree new_temp = NULL_TREE;
8135 18844 : auto_vec<tree> vec_oprnds[3];
8136 :
8137 2692 : if (dump_enabled_p ())
8138 807 : dump_printf_loc (MSG_NOTE, vect_location, "transform reduction.\n");
8139 :
8140 : /* A binary COND_OP reduction must have the same definition and else
8141 : value. */
8142 3176 : bool cond_fn_p = code.is_internal_fn ()
8143 484 : && conditional_internal_fn_code (internal_fn (code)) != ERROR_MARK;
8144 482 : if (cond_fn_p)
8145 : {
8146 482 : gcc_assert (code == IFN_COND_ADD || code == IFN_COND_SUB
8147 : || code == IFN_COND_MUL || code == IFN_COND_AND
8148 : || code == IFN_COND_IOR || code == IFN_COND_XOR
8149 : || code == IFN_COND_MIN || code == IFN_COND_MAX);
8150 482 : gcc_assert (op.num_ops == 4
8151 : && (op.ops[reduc_index]
8152 : == op.ops[internal_fn_else_index ((internal_fn) code)]));
8153 : }
8154 :
8155 2692 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
8156 :
8157 2692 : vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
8158 2692 : if (reduction_type == FOLD_LEFT_REDUCTION)
8159 : {
8160 977 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
8161 977 : gcc_assert (code.is_tree_code () || cond_fn_p);
8162 977 : return vectorize_fold_left_reduction
8163 977 : (loop_vinfo, stmt_info, gsi, slp_node,
8164 977 : code, reduc_fn, op.num_ops, vectype_in,
8165 977 : reduc_index, masks, lens);
8166 : }
8167 :
8168 1715 : bool single_defuse_cycle = VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info);
8169 1715 : bool lane_reducing = lane_reducing_op_p (code);
8170 1434 : gcc_assert (single_defuse_cycle || lane_reducing);
8171 :
8172 1715 : if (lane_reducing)
8173 : {
8174 : /* The last operand of lane-reducing op is for reduction. */
8175 281 : gcc_assert (reduc_index == (int) op.num_ops - 1);
8176 : }
8177 :
8178 : /* Create the destination vector */
8179 1715 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
8180 1715 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
8181 :
8182 : /* Get NCOPIES vector definitions for all operands except the reduction
8183 : definition. */
8184 1715 : if (!cond_fn_p)
8185 : {
8186 1262 : gcc_assert (reduc_index >= 0 && reduc_index <= 2);
8187 1262 : vect_get_vec_defs (loop_vinfo, slp_node,
8188 1262 : single_defuse_cycle && reduc_index == 0
8189 1262 : ? NULL_TREE : op.ops[0], &vec_oprnds[0],
8190 1262 : single_defuse_cycle && reduc_index == 1
8191 1262 : ? NULL_TREE : op.ops[1], &vec_oprnds[1],
8192 1262 : op.num_ops == 3
8193 283 : && !(single_defuse_cycle && reduc_index == 2)
8194 1369 : ? op.ops[2] : NULL_TREE, &vec_oprnds[2]);
8195 : }
8196 : else
8197 : {
8198 : /* For a conditional operation pass the truth type as mask
8199 : vectype. */
8200 453 : gcc_assert (single_defuse_cycle
8201 : && (reduc_index == 1 || reduc_index == 2));
8202 453 : vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
8203 : &vec_oprnds[0],
8204 2 : reduc_index == 1 ? NULL_TREE : op.ops[1],
8205 : &vec_oprnds[1],
8206 453 : reduc_index == 2 ? NULL_TREE : op.ops[2],
8207 : &vec_oprnds[2]);
8208 : }
8209 :
8210 : /* For single def-use cycles get one copy of the vectorized reduction
8211 : definition. */
8212 3017 : unsigned vec_in_num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
8213 1715 : if (single_defuse_cycle)
8214 : {
8215 1608 : vect_get_vec_defs (loop_vinfo, slp_node,
8216 1608 : reduc_index == 0 ? op.ops[0] : NULL_TREE,
8217 : &vec_oprnds[0],
8218 1608 : reduc_index == 1 ? op.ops[1] : NULL_TREE,
8219 : &vec_oprnds[1],
8220 1608 : reduc_index == 2 ? op.ops[2] : NULL_TREE,
8221 : &vec_oprnds[2]);
8222 : }
8223 107 : else if (lane_reducing)
8224 : {
8225 : /* For normal reduction, consistency between vectorized def/use is
8226 : naturally ensured when mapping from scalar statement. But if lane-
8227 : reducing op is involved in reduction, thing would become somewhat
8228 : complicated in that the op's result and operand for accumulation are
8229 : limited to less lanes than other operands, which certainly causes
8230 : def/use mismatch on adjacent statements around the op if do not have
8231 : any kind of specific adjustment. One approach is to refit lane-
8232 : reducing op in the way of introducing new trivial pass-through copies
8233 : to fix possible def/use gap, so as to make it behave like a normal op.
8234 : And vector reduction PHIs are always generated to the full extent, no
8235 : matter lane-reducing op exists or not. If some copies or PHIs are
8236 : actually superfluous, they would be cleaned up by passes after
8237 : vectorization. An example for single-lane slp, lane-reducing ops
8238 : with mixed input vectypes in a reduction chain, is given as below.
8239 : Similarly, this handling is applicable for multiple-lane slp as well.
8240 :
8241 : int sum = 1;
8242 : for (i)
8243 : {
8244 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
8245 : sum += w[i]; // widen-sum <vector(16) char>
8246 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
8247 : sum += n[i]; // normal <vector(4) int>
8248 : }
8249 :
8250 : The vector size is 128-bit,vectorization factor is 16. Reduction
8251 : statements would be transformed as:
8252 :
8253 : vector<4> int sum_v0 = { 0, 0, 0, 1 };
8254 : vector<4> int sum_v1 = { 0, 0, 0, 0 };
8255 : vector<4> int sum_v2 = { 0, 0, 0, 0 };
8256 : vector<4> int sum_v3 = { 0, 0, 0, 0 };
8257 :
8258 : for (i / 16)
8259 : {
8260 : sum_v0 = DOT_PROD (d0_v0[i: 0 ~ 15], d1_v0[i: 0 ~ 15], sum_v0);
8261 : sum_v1 = sum_v1; // copy
8262 : sum_v2 = sum_v2; // copy
8263 : sum_v3 = sum_v3; // copy
8264 :
8265 : sum_v0 = sum_v0; // copy
8266 : sum_v1 = WIDEN_SUM (w_v1[i: 0 ~ 15], sum_v1);
8267 : sum_v2 = sum_v2; // copy
8268 : sum_v3 = sum_v3; // copy
8269 :
8270 : sum_v0 = sum_v0; // copy
8271 : sum_v1 = SAD (s0_v1[i: 0 ~ 7 ], s1_v1[i: 0 ~ 7 ], sum_v1);
8272 : sum_v2 = SAD (s0_v2[i: 8 ~ 15], s1_v2[i: 8 ~ 15], sum_v2);
8273 : sum_v3 = sum_v3; // copy
8274 :
8275 : sum_v0 += n_v0[i: 0 ~ 3 ];
8276 : sum_v1 += n_v1[i: 4 ~ 7 ];
8277 : sum_v2 += n_v2[i: 8 ~ 11];
8278 : sum_v3 += n_v3[i: 12 ~ 15];
8279 : }
8280 :
8281 : Moreover, for a higher instruction parallelism in final vectorized
8282 : loop, it is considered to make those effective vector lane-reducing
8283 : ops be distributed evenly among all def-use cycles. In the above
8284 : example, DOT_PROD, WIDEN_SUM and SADs are generated into disparate
8285 : cycles, instruction dependency among them could be eliminated. */
8286 107 : unsigned effec_ncopies = vec_oprnds[0].length ();
8287 107 : unsigned total_ncopies = vec_oprnds[reduc_index].length ();
8288 :
8289 107 : gcc_assert (effec_ncopies <= total_ncopies);
8290 :
8291 107 : if (effec_ncopies < total_ncopies)
8292 : {
8293 294 : for (unsigned i = 0; i < op.num_ops - 1; i++)
8294 : {
8295 392 : gcc_assert (vec_oprnds[i].length () == effec_ncopies);
8296 196 : vec_oprnds[i].safe_grow_cleared (total_ncopies);
8297 : }
8298 : }
8299 :
8300 107 : tree reduc_vectype_in = vectype_in;
8301 107 : gcc_assert (reduc_vectype_in);
8302 : }
8303 :
8304 1715 : bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (slp_node);
8305 1715 : unsigned num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
8306 1715 : unsigned mask_index = 0;
8307 :
8308 7484 : for (unsigned i = 0; i < num; ++i)
8309 : {
8310 5769 : gimple *new_stmt;
8311 5769 : tree vop[3] = { vec_oprnds[0][i], vec_oprnds[1][i],
8312 8076 : op.num_ops >= 3 ? vec_oprnds[2][i] : NULL_TREE };
8313 5769 : if (!vop[0] || !vop[1])
8314 : {
8315 503 : tree reduc_vop = vec_oprnds[reduc_index][i];
8316 :
8317 : /* If could not generate an effective vector statement for current
8318 : portion of reduction operand, insert a trivial copy to simply
8319 : handle over the operand to other dependent statements. */
8320 503 : gcc_assert (reduc_vop);
8321 :
8322 503 : if (TREE_CODE (reduc_vop) == SSA_NAME
8323 503 : && !SSA_NAME_IS_DEFAULT_DEF (reduc_vop))
8324 503 : new_stmt = SSA_NAME_DEF_STMT (reduc_vop);
8325 : else
8326 : {
8327 0 : new_temp = make_ssa_name (vec_dest);
8328 0 : new_stmt = gimple_build_assign (new_temp, reduc_vop);
8329 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
8330 : gsi);
8331 : }
8332 : }
8333 5266 : else if (masked_loop_p && !mask_by_cond_expr)
8334 : {
8335 : /* No conditional ifns have been defined for lane-reducing op
8336 : yet. */
8337 20 : gcc_assert (!lane_reducing);
8338 :
8339 32 : auto_vec<tree> args (op.num_ops + (cond_fn_p ? 0 : 2));
8340 20 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8341 : vec_in_num, vectype_in,
8342 20 : mask_index++);
8343 20 : if (cond_fn_p)
8344 : {
8345 8 : gcc_assert (internal_fn_mask_index (internal_fn (code)) == 0);
8346 8 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask),
8347 : mask, vop[0], gsi);
8348 : }
8349 20 : gcc_assert (internal_fn_mask_index (internal_fn (cond_fn)) == 0);
8350 20 : args.quick_push (mask);
8351 20 : for (unsigned j = cond_fn_p ? 1 : 0;
8352 104 : j < op.num_ops - (cond_fn_p ? 1 : 0); ++j)
8353 44 : args.quick_push (vop[j]);
8354 40 : gcc_assert (internal_fn_else_index (internal_fn (cond_fn))
8355 : == (int) args.length ());
8356 20 : args.quick_push (vop[reduc_index]);
8357 20 : gcall *call = gimple_build_call_internal_vec (cond_fn, args);
8358 20 : new_temp = make_ssa_name (vec_dest, call);
8359 20 : gimple_call_set_lhs (call, new_temp);
8360 20 : gimple_call_set_nothrow (call, true);
8361 20 : vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi);
8362 20 : new_stmt = call;
8363 20 : }
8364 : else
8365 : {
8366 5246 : if (masked_loop_p && mask_by_cond_expr)
8367 : {
8368 4 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8369 : vec_in_num, vectype_in,
8370 : mask_index++);
8371 4 : build_vect_cond_expr (code, vop, mask, gsi);
8372 : }
8373 :
8374 5246 : if (emulated_mixed_dot_prod)
8375 4 : new_stmt = vect_emulate_mixed_dot_prod (loop_vinfo, stmt_info, gsi,
8376 : vec_dest, vop);
8377 :
8378 6584 : else if (code.is_internal_fn () && !cond_fn_p)
8379 0 : new_stmt = gimple_build_call_internal (internal_fn (code),
8380 : op.num_ops,
8381 : vop[0], vop[1], vop[2]);
8382 6584 : else if (code.is_internal_fn () && cond_fn_p)
8383 1342 : new_stmt = gimple_build_call_internal (internal_fn (code),
8384 : op.num_ops,
8385 : vop[0], vop[1], vop[2],
8386 : vop[reduc_index]);
8387 : else
8388 3900 : new_stmt = gimple_build_assign (vec_dest, tree_code (op.code),
8389 : vop[0], vop[1], vop[2]);
8390 5246 : new_temp = make_ssa_name (vec_dest, new_stmt);
8391 5246 : gimple_set_lhs (new_stmt, new_temp);
8392 5246 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8393 : }
8394 :
8395 5769 : if (single_defuse_cycle && i < num - 1)
8396 3392 : vec_oprnds[reduc_index].safe_push (gimple_get_lhs (new_stmt));
8397 : else
8398 2377 : slp_node->push_vec_def (new_stmt);
8399 : }
8400 :
8401 : return true;
8402 10768 : }
8403 :
8404 : /* Transform phase of a cycle PHI. */
8405 :
8406 : bool
8407 23672 : vect_transform_cycle_phi (loop_vec_info loop_vinfo,
8408 : stmt_vec_info stmt_info,
8409 : slp_tree slp_node, slp_instance slp_node_instance)
8410 : {
8411 23672 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
8412 23672 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
8413 23672 : int i;
8414 23672 : bool nested_cycle = false;
8415 23672 : int vec_num;
8416 :
8417 23672 : if (nested_in_vect_loop_p (loop, stmt_info))
8418 : {
8419 23672 : loop = loop->inner;
8420 23672 : nested_cycle = true;
8421 : }
8422 :
8423 23672 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
8424 23672 : if (reduc_info
8425 23089 : && (VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
8426 23089 : || VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION))
8427 : /* Leave the scalar phi in place. */
8428 : return true;
8429 :
8430 22112 : if (reduc_info && reduc_info->is_reduc_chain && dump_enabled_p ())
8431 133 : dump_printf_loc (MSG_NOTE, vect_location,
8432 : "vectorizing a reduction chain\n");
8433 :
8434 22695 : vec_num = vect_get_num_copies (loop_vinfo, slp_node);
8435 :
8436 : /* Check whether we should use a single PHI node and accumulate
8437 : vectors to one before the backedge. */
8438 22695 : if (reduc_info && VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info))
8439 22695 : vec_num = 1;
8440 :
8441 : /* Create the destination vector */
8442 22695 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
8443 22695 : tree vec_dest = vect_create_destination_var (gimple_phi_result (phi),
8444 : vectype_out);
8445 :
8446 : /* Get the loop-entry arguments. */
8447 22695 : auto_vec<tree> vec_initial_defs;
8448 22695 : vec_initial_defs.reserve (vec_num);
8449 : /* Optimize: if initial_def is for REDUC_MAX smaller than the base
8450 : and we can't use zero for induc_val, use initial_def. Similarly
8451 : for REDUC_MIN and initial_def larger than the base. */
8452 22695 : if (reduc_info
8453 22112 : && VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
8454 : {
8455 62 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8456 62 : tree initial_def = vect_phi_initial_value (phi);
8457 62 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).safe_push (initial_def);
8458 62 : tree induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
8459 62 : if (TREE_CODE (initial_def) == INTEGER_CST
8460 60 : && !integer_zerop (induc_val)
8461 122 : && ((VECT_REDUC_INFO_CODE (reduc_info) == MAX_EXPR
8462 42 : && tree_int_cst_lt (initial_def, induc_val))
8463 58 : || (VECT_REDUC_INFO_CODE (reduc_info) == MIN_EXPR
8464 18 : && tree_int_cst_lt (induc_val, initial_def))))
8465 : {
8466 2 : induc_val = initial_def;
8467 : /* Communicate we used the initial_def to epilouge
8468 : generation. */
8469 2 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
8470 : }
8471 62 : vec_initial_defs.quick_push
8472 62 : (build_vector_from_val (vectype_out, induc_val));
8473 62 : }
8474 22633 : else if (nested_cycle)
8475 : {
8476 663 : unsigned phi_idx = loop_preheader_edge (loop)->dest_idx;
8477 663 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[phi_idx],
8478 : &vec_initial_defs);
8479 : }
8480 : else
8481 : {
8482 21970 : gcc_assert (slp_node == slp_node_instance->reduc_phis);
8483 21970 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
8484 21970 : vec<stmt_vec_info> &stmts = SLP_TREE_SCALAR_STMTS (slp_node);
8485 :
8486 21970 : unsigned int num_phis = stmts.length ();
8487 21970 : if (reduc_info->is_reduc_chain)
8488 215 : num_phis = 1;
8489 21970 : initial_values.reserve (num_phis);
8490 66387 : for (unsigned int i = 0; i < num_phis; ++i)
8491 : {
8492 22447 : gphi *this_phi = as_a<gphi *> (stmts[i]->stmt);
8493 22447 : initial_values.quick_push (vect_phi_initial_value (this_phi));
8494 : }
8495 21970 : tree neutral_op = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
8496 21970 : if (vec_num == 1
8497 21970 : && vect_find_reusable_accumulator (loop_vinfo,
8498 : reduc_info, vectype_out))
8499 : ;
8500 : /* Try to simplify the vector initialization by applying an
8501 : adjustment after the reduction has been performed. This
8502 : can also break a critical path but on the other hand
8503 : requires to keep the initial value live across the loop. */
8504 18033 : else if (neutral_op
8505 17444 : && initial_values.length () == 1
8506 17235 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
8507 35197 : && !operand_equal_p (neutral_op, initial_values[0]))
8508 : {
8509 12139 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info)
8510 12139 : = initial_values[0];
8511 12139 : initial_values[0] = neutral_op;
8512 : }
8513 21970 : if (!VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
8514 3937 : || loop_vinfo->main_loop_edge)
8515 43642 : get_initial_defs_for_reduction (loop_vinfo, reduc_info, vectype_out,
8516 : &vec_initial_defs, vec_num,
8517 : stmts.length (), neutral_op);
8518 : }
8519 :
8520 22695 : if (reduc_info)
8521 22112 : if (auto *accumulator = VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
8522 : {
8523 3937 : tree def = accumulator->reduc_input;
8524 3937 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8525 : {
8526 3934 : unsigned int nreduc;
8527 7868 : bool res = constant_multiple_p (TYPE_VECTOR_SUBPARTS
8528 3934 : (TREE_TYPE (def)),
8529 3934 : TYPE_VECTOR_SUBPARTS (vectype_out),
8530 : &nreduc);
8531 0 : gcc_assert (res);
8532 3934 : gimple_seq stmts = NULL;
8533 : /* Reduce the single vector to a smaller one. */
8534 3934 : if (nreduc != 1)
8535 : {
8536 : /* Perform the reduction in the appropriate type. */
8537 3934 : tree rvectype = vectype_out;
8538 3934 : if (!useless_type_conversion_p (TREE_TYPE (vectype_out),
8539 3934 : TREE_TYPE (TREE_TYPE (def))))
8540 235 : rvectype = build_vector_type (TREE_TYPE (TREE_TYPE (def)),
8541 : TYPE_VECTOR_SUBPARTS
8542 470 : (vectype_out));
8543 3934 : def = vect_create_partial_epilog (def, rvectype,
8544 : VECT_REDUC_INFO_CODE
8545 : (reduc_info),
8546 : &stmts);
8547 : }
8548 : /* The epilogue loop might use a different vector mode, like
8549 : VNx2DI vs. V2DI. */
8550 3934 : if (TYPE_MODE (vectype_out) != TYPE_MODE (TREE_TYPE (def)))
8551 : {
8552 0 : tree reduc_type = build_vector_type_for_mode
8553 0 : (TREE_TYPE (TREE_TYPE (def)), TYPE_MODE (vectype_out));
8554 0 : def = gimple_convert (&stmts, reduc_type, def);
8555 : }
8556 : /* Adjust the input so we pick up the partially reduced value
8557 : for the skip edge in vect_create_epilog_for_reduction. */
8558 3934 : accumulator->reduc_input = def;
8559 : /* And the reduction could be carried out using a different sign. */
8560 3934 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8561 235 : def = gimple_convert (&stmts, vectype_out, def);
8562 3934 : edge e;
8563 3934 : if ((e = loop_vinfo->main_loop_edge)
8564 3934 : || (e = loop_vinfo->skip_this_loop_edge))
8565 : {
8566 : /* While we'd like to insert on the edge this will split
8567 : blocks and disturb bookkeeping, we also will eventually
8568 : need this on the skip edge. Rely on sinking to
8569 : fixup optimal placement and insert in the pred. */
8570 3785 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
8571 : /* Insert before a cond that eventually skips the
8572 : epilogue. */
8573 3785 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
8574 3768 : gsi_prev (&gsi);
8575 3785 : gsi_insert_seq_after (&gsi, stmts, GSI_CONTINUE_LINKING);
8576 : }
8577 : else
8578 149 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop),
8579 : stmts);
8580 : }
8581 3937 : if (loop_vinfo->main_loop_edge)
8582 3788 : vec_initial_defs[0]
8583 3788 : = vect_get_main_loop_result (loop_vinfo, def,
8584 3788 : vec_initial_defs[0]);
8585 : else
8586 149 : vec_initial_defs.safe_push (def);
8587 : }
8588 :
8589 : /* Generate the reduction PHIs upfront. */
8590 47244 : for (i = 0; i < vec_num; i++)
8591 : {
8592 24549 : tree vec_init_def = vec_initial_defs[i];
8593 : /* Create the reduction-phi that defines the reduction
8594 : operand. */
8595 24549 : gphi *new_phi = create_phi_node (vec_dest, loop->header);
8596 24549 : add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
8597 : UNKNOWN_LOCATION);
8598 :
8599 : /* The loop-latch arg is set in epilogue processing. */
8600 :
8601 24549 : slp_node->push_vec_def (new_phi);
8602 : }
8603 :
8604 22695 : return true;
8605 22695 : }
8606 :
8607 : /* Vectorizes LC PHIs. */
8608 :
8609 : bool
8610 206990 : vectorizable_lc_phi (loop_vec_info loop_vinfo,
8611 : stmt_vec_info stmt_info,
8612 : slp_tree slp_node)
8613 : {
8614 206990 : if (!loop_vinfo
8615 206990 : || !is_a <gphi *> (stmt_info->stmt)
8616 251863 : || gimple_phi_num_args (stmt_info->stmt) != 1)
8617 : return false;
8618 :
8619 805 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8620 0 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
8621 : return false;
8622 :
8623 : /* Deal with copies from externs or constants that disguise as
8624 : loop-closed PHI nodes (PR97886). */
8625 805 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
8626 : SLP_TREE_VECTYPE (slp_node)))
8627 : {
8628 0 : if (dump_enabled_p ())
8629 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8630 : "incompatible vector types for invariants\n");
8631 : return false;
8632 : }
8633 :
8634 : /* ??? This can happen with data vs. mask uses of boolean. */
8635 805 : if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
8636 805 : SLP_TREE_VECTYPE
8637 : (SLP_TREE_CHILDREN (slp_node)[0])))
8638 : {
8639 0 : if (dump_enabled_p ())
8640 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8641 : "missed mask promotion\n");
8642 : return false;
8643 : }
8644 :
8645 805 : SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
8646 805 : return true;
8647 : }
8648 :
8649 : bool
8650 489 : vect_transform_lc_phi (loop_vec_info loop_vinfo,
8651 : stmt_vec_info stmt_info,
8652 : slp_tree slp_node)
8653 : {
8654 :
8655 489 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8656 489 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8657 489 : basic_block bb = gimple_bb (stmt_info->stmt);
8658 489 : edge e = single_pred_edge (bb);
8659 489 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8660 489 : auto_vec<tree> vec_oprnds;
8661 489 : vect_get_vec_defs (loop_vinfo, slp_node, true, &vec_oprnds);
8662 1577 : for (unsigned i = 0; i < vec_oprnds.length (); i++)
8663 : {
8664 : /* Create the vectorized LC PHI node. */
8665 599 : gphi *new_phi = create_phi_node (vec_dest, bb);
8666 599 : add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
8667 599 : slp_node->push_vec_def (new_phi);
8668 : }
8669 :
8670 489 : return true;
8671 489 : }
8672 :
8673 : /* Vectorizes PHIs. */
8674 :
8675 : bool
8676 153900 : vectorizable_phi (bb_vec_info vinfo,
8677 : stmt_vec_info stmt_info,
8678 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8679 : {
8680 153900 : if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
8681 : return false;
8682 :
8683 73823 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
8684 : return false;
8685 :
8686 73823 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8687 :
8688 73823 : if (cost_vec) /* transformation not required. */
8689 : {
8690 : slp_tree child;
8691 : unsigned i;
8692 206347 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), i, child)
8693 145286 : if (!child)
8694 : {
8695 0 : if (dump_enabled_p ())
8696 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8697 : "PHI node with unvectorized backedge def\n");
8698 : return false;
8699 : }
8700 145286 : else if (!vect_maybe_update_slp_op_vectype (child, vectype))
8701 : {
8702 26 : if (dump_enabled_p ())
8703 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8704 : "incompatible vector types for invariants\n");
8705 : return false;
8706 : }
8707 145260 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8708 145260 : && !useless_type_conversion_p (vectype,
8709 : SLP_TREE_VECTYPE (child)))
8710 : {
8711 : /* With bools we can have mask and non-mask precision vectors
8712 : or different non-mask precisions. while pattern recog is
8713 : supposed to guarantee consistency here bugs in it can cause
8714 : mismatches (PR103489 and PR103800 for example).
8715 : Deal with them here instead of ICEing later. */
8716 18 : if (dump_enabled_p ())
8717 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8718 : "incompatible vector type setup from "
8719 : "bool pattern detection\n");
8720 : return false;
8721 : }
8722 :
8723 : /* For single-argument PHIs assume coalescing which means zero cost
8724 : for the scalar and the vector PHIs. This avoids artificially
8725 : favoring the vector path (but may pessimize it in some cases). */
8726 61061 : if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
8727 55307 : record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, slp_node),
8728 : vector_stmt, slp_node, vectype, 0, vect_body);
8729 61061 : SLP_TREE_TYPE (slp_node) = phi_info_type;
8730 61061 : return true;
8731 : }
8732 :
8733 12718 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8734 12718 : basic_block bb = gimple_bb (stmt_info->stmt);
8735 12718 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8736 12718 : auto_vec<gphi *> new_phis;
8737 44889 : for (unsigned i = 0; i < gimple_phi_num_args (stmt_info->stmt); ++i)
8738 : {
8739 32171 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
8740 :
8741 : /* Skip not yet vectorized defs. */
8742 32760 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8743 32171 : && SLP_TREE_VEC_DEFS (child).is_empty ())
8744 589 : continue;
8745 :
8746 31582 : auto_vec<tree> vec_oprnds;
8747 31582 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[i], &vec_oprnds);
8748 31582 : if (!new_phis.exists ())
8749 : {
8750 12718 : new_phis.create (vec_oprnds.length ());
8751 39433 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8752 : {
8753 : /* Create the vectorized LC PHI node. */
8754 13997 : new_phis.quick_push (create_phi_node (vec_dest, bb));
8755 13997 : slp_node->push_vec_def (new_phis[j]);
8756 : }
8757 : }
8758 31582 : edge e = gimple_phi_arg_edge (as_a <gphi *> (stmt_info->stmt), i);
8759 67447 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8760 35865 : add_phi_arg (new_phis[j], vec_oprnds[j], e, UNKNOWN_LOCATION);
8761 31582 : }
8762 : /* We should have at least one already vectorized child. */
8763 12718 : gcc_assert (new_phis.exists ());
8764 :
8765 12718 : return true;
8766 12718 : }
8767 :
8768 : /* Vectorizes first order recurrences. An overview of the transformation
8769 : is described below. Suppose we have the following loop.
8770 :
8771 : int t = 0;
8772 : for (int i = 0; i < n; ++i)
8773 : {
8774 : b[i] = a[i] - t;
8775 : t = a[i];
8776 : }
8777 :
8778 : There is a first-order recurrence on 'a'. For this loop, the scalar IR
8779 : looks (simplified) like:
8780 :
8781 : scalar.preheader:
8782 : init = 0;
8783 :
8784 : scalar.body:
8785 : i = PHI <0(scalar.preheader), i+1(scalar.body)>
8786 : _2 = PHI <(init(scalar.preheader), <_1(scalar.body)>
8787 : _1 = a[i]
8788 : b[i] = _1 - _2
8789 : if (i < n) goto scalar.body
8790 :
8791 : In this example, _2 is a recurrence because it's value depends on the
8792 : previous iteration. We vectorize this as (VF = 4)
8793 :
8794 : vector.preheader:
8795 : vect_init = vect_cst(..., ..., ..., 0)
8796 :
8797 : vector.body
8798 : i = PHI <0(vector.preheader), i+4(vector.body)>
8799 : vect_1 = PHI <vect_init(vector.preheader), v2(vector.body)>
8800 : vect_2 = a[i, i+1, i+2, i+3];
8801 : vect_3 = vec_perm (vect_1, vect_2, { 3, 4, 5, 6 })
8802 : b[i, i+1, i+2, i+3] = vect_2 - vect_3
8803 : if (..) goto vector.body
8804 :
8805 : In this function, vectorizable_recurr, we code generate both the
8806 : vector PHI node and the permute since those together compute the
8807 : vectorized value of the scalar PHI. We do not yet have the
8808 : backedge value to fill in there nor into the vec_perm. Those
8809 : are filled in vect_schedule_scc.
8810 :
8811 : TODO: Since the scalar loop does not have a use of the recurrence
8812 : outside of the loop the natural way to implement peeling via
8813 : vectorizing the live value doesn't work. For now peeling of loops
8814 : with a recurrence is not implemented. For SLP the supported cases
8815 : are restricted to those requiring a single vector recurrence PHI. */
8816 :
8817 : bool
8818 206228 : vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
8819 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8820 : {
8821 206228 : if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
8822 : return false;
8823 :
8824 44111 : gphi *phi = as_a<gphi *> (stmt_info->stmt);
8825 :
8826 : /* So far we only support first-order recurrence auto-vectorization. */
8827 44111 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
8828 : return false;
8829 :
8830 424 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8831 424 : unsigned ncopies = vect_get_num_copies (loop_vinfo, slp_node);
8832 424 : poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8833 424 : unsigned dist = SLP_TREE_LANES (slp_node);
8834 : /* We need to be able to make progress with a single vector. */
8835 424 : if (maybe_gt (dist * 2, nunits))
8836 : {
8837 0 : if (dump_enabled_p ())
8838 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8839 : "first order recurrence exceeds half of "
8840 : "a vector\n");
8841 : return false;
8842 : }
8843 :
8844 : /* We need to be able to build a { ..., a, b } init vector with
8845 : dist number of distinct trailing values. Always possible
8846 : when dist == 1 or when nunits is constant or when the initializations
8847 : are uniform. */
8848 424 : tree uniform_initval = NULL_TREE;
8849 424 : edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
8850 1720 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8851 : {
8852 460 : gphi *phi = as_a <gphi *> (s->stmt);
8853 460 : if (! uniform_initval)
8854 424 : uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8855 36 : else if (! operand_equal_p (uniform_initval,
8856 36 : PHI_ARG_DEF_FROM_EDGE (phi, pe)))
8857 : {
8858 : uniform_initval = NULL_TREE;
8859 : break;
8860 : }
8861 : }
8862 424 : if (!uniform_initval && !nunits.is_constant ())
8863 : {
8864 : if (dump_enabled_p ())
8865 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8866 : "cannot build initialization vector for "
8867 : "first order recurrence\n");
8868 : return false;
8869 : }
8870 :
8871 : /* First-order recurrence autovectorization needs to handle permutation
8872 : with indices = [nunits-1, nunits, nunits+1, ...]. */
8873 424 : vec_perm_builder sel (nunits, 1, 3);
8874 1696 : for (int i = 0; i < 3; ++i)
8875 1272 : sel.quick_push (nunits - dist + i);
8876 424 : vec_perm_indices indices (sel, 2, nunits);
8877 :
8878 424 : if (cost_vec) /* transformation not required. */
8879 : {
8880 381 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
8881 : indices))
8882 : return false;
8883 :
8884 : /* We eventually need to set a vector type on invariant
8885 : arguments. */
8886 : unsigned j;
8887 : slp_tree child;
8888 807 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
8889 538 : if (!vect_maybe_update_slp_op_vectype (child, vectype))
8890 : {
8891 0 : if (dump_enabled_p ())
8892 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8893 : "incompatible vector types for "
8894 : "invariants\n");
8895 : return false;
8896 : }
8897 :
8898 : /* Verify we have set up compatible types. */
8899 269 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8900 269 : slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
8901 269 : tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
8902 269 : if (!types_compatible_p (latch_vectype, vectype))
8903 : return false;
8904 :
8905 : /* The recurrence costs the initialization vector and one permute
8906 : for each copy. With SLP the prologue value is explicitly
8907 : represented and costed separately. */
8908 269 : unsigned prologue_cost = 0;
8909 269 : unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
8910 : slp_node, 0, vect_body);
8911 269 : if (dump_enabled_p ())
8912 51 : dump_printf_loc (MSG_NOTE, vect_location,
8913 : "vectorizable_recurr: inside_cost = %d, "
8914 : "prologue_cost = %d .\n", inside_cost,
8915 : prologue_cost);
8916 :
8917 269 : SLP_TREE_TYPE (slp_node) = recurr_info_type;
8918 269 : return true;
8919 : }
8920 :
8921 43 : tree vec_init;
8922 43 : if (! uniform_initval)
8923 : {
8924 6 : vec<constructor_elt, va_gc> *v = NULL;
8925 6 : vec_alloc (v, nunits.to_constant ());
8926 39 : for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
8927 27 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
8928 : build_zero_cst (TREE_TYPE (vectype)));
8929 39 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8930 : {
8931 21 : gphi *phi = as_a <gphi *> (s->stmt);
8932 21 : tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8933 21 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
8934 21 : TREE_TYPE (preheader)))
8935 : {
8936 0 : gimple_seq stmts = NULL;
8937 0 : preheader = gimple_convert (&stmts,
8938 0 : TREE_TYPE (vectype), preheader);
8939 0 : gsi_insert_seq_on_edge_immediate (pe, stmts);
8940 : }
8941 21 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
8942 : }
8943 6 : vec_init = build_constructor (vectype, v);
8944 : }
8945 : else
8946 : vec_init = uniform_initval;
8947 43 : vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
8948 :
8949 : /* Create the vectorized first-order PHI node. */
8950 43 : tree vec_dest = vect_get_new_vect_var (vectype,
8951 : vect_simple_var, "vec_recur_");
8952 43 : basic_block bb = gimple_bb (phi);
8953 43 : gphi *new_phi = create_phi_node (vec_dest, bb);
8954 43 : add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
8955 :
8956 : /* Insert shuffles the first-order recurrence autovectorization.
8957 : result = VEC_PERM <vec_recur, vect_1, index[nunits-1, nunits, ...]>. */
8958 43 : tree perm = vect_gen_perm_mask_checked (vectype, indices);
8959 :
8960 : /* Insert the required permute after the latch definition. The
8961 : second and later operands are tentative and will be updated when we have
8962 : vectorized the latch definition. */
8963 43 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8964 43 : gimple *latch_def = SSA_NAME_DEF_STMT (PHI_ARG_DEF_FROM_EDGE (phi, le));
8965 43 : gimple_stmt_iterator gsi2 = gsi_for_stmt (latch_def);
8966 49 : do
8967 : {
8968 49 : gsi_next (&gsi2);
8969 : }
8970 : /* Skip inserted vectorized stmts for the latch definition. We have to
8971 : insert after those. */
8972 92 : while (gsi_stmt (gsi2) && gimple_uid (gsi_stmt (gsi2)) == 0);
8973 :
8974 123 : for (unsigned i = 0; i < ncopies; ++i)
8975 : {
8976 80 : vec_dest = make_ssa_name (vectype);
8977 80 : gassign *vperm
8978 123 : = gimple_build_assign (vec_dest, VEC_PERM_EXPR,
8979 43 : i == 0 ? gimple_phi_result (new_phi) : NULL,
8980 : NULL, perm);
8981 80 : vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
8982 :
8983 80 : slp_node->push_vec_def (vperm);
8984 : }
8985 :
8986 : return true;
8987 424 : }
8988 :
8989 : /* Return true if VECTYPE represents a vector that requires lowering
8990 : by the vector lowering pass. */
8991 :
8992 : bool
8993 680116 : vect_emulated_vector_p (tree vectype)
8994 : {
8995 1360232 : return (!VECTOR_MODE_P (TYPE_MODE (vectype))
8996 684246 : && (!VECTOR_BOOLEAN_TYPE_P (vectype)
8997 4112 : || TYPE_PRECISION (TREE_TYPE (vectype)) != 1));
8998 : }
8999 :
9000 : /* Return true if we can emulate CODE on an integer mode representation
9001 : of a vector. */
9002 :
9003 : bool
9004 12364 : vect_can_vectorize_without_simd_p (tree_code code)
9005 : {
9006 12364 : switch (code)
9007 : {
9008 : case PLUS_EXPR:
9009 : case MINUS_EXPR:
9010 : case NEGATE_EXPR:
9011 : case BIT_AND_EXPR:
9012 : case BIT_IOR_EXPR:
9013 : case BIT_XOR_EXPR:
9014 : case BIT_NOT_EXPR:
9015 : return true;
9016 :
9017 11754 : default:
9018 11754 : return false;
9019 : }
9020 : }
9021 :
9022 : /* Likewise, but taking a code_helper. */
9023 :
9024 : bool
9025 1003 : vect_can_vectorize_without_simd_p (code_helper code)
9026 : {
9027 1003 : return (code.is_tree_code ()
9028 1003 : && vect_can_vectorize_without_simd_p (tree_code (code)));
9029 : }
9030 :
9031 : /* Create vector init for vectorized iv. */
9032 : static tree
9033 932 : vect_create_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
9034 : tree step_expr, poly_uint64 nunits,
9035 : tree vectype,
9036 : enum vect_induction_op_type induction_type)
9037 : {
9038 932 : unsigned HOST_WIDE_INT const_nunits;
9039 932 : tree vec_shift, vec_init, new_name;
9040 932 : unsigned i;
9041 932 : tree itype = TREE_TYPE (vectype);
9042 :
9043 : /* iv_loop is the loop to be vectorized. Create:
9044 : vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr). */
9045 932 : new_name = gimple_convert (stmts, itype, init_expr);
9046 932 : switch (induction_type)
9047 : {
9048 21 : case vect_step_op_shr:
9049 21 : case vect_step_op_shl:
9050 : /* Build the Initial value from shift_expr. */
9051 21 : vec_init = gimple_build_vector_from_val (stmts,
9052 : vectype,
9053 : new_name);
9054 21 : vec_shift = gimple_build (stmts, VEC_SERIES_EXPR, vectype,
9055 : build_zero_cst (itype), step_expr);
9056 21 : vec_init = gimple_build (stmts,
9057 : (induction_type == vect_step_op_shr
9058 : ? RSHIFT_EXPR : LSHIFT_EXPR),
9059 : vectype, vec_init, vec_shift);
9060 21 : break;
9061 :
9062 833 : case vect_step_op_neg:
9063 833 : {
9064 833 : vec_init = gimple_build_vector_from_val (stmts,
9065 : vectype,
9066 : new_name);
9067 833 : tree vec_neg = gimple_build (stmts, NEGATE_EXPR,
9068 : vectype, vec_init);
9069 : /* The encoding has 2 interleaved stepped patterns. */
9070 833 : vec_perm_builder sel (nunits, 2, 3);
9071 833 : sel.quick_grow (6);
9072 4165 : for (i = 0; i < 3; i++)
9073 : {
9074 2499 : sel[2 * i] = i;
9075 2499 : sel[2 * i + 1] = i + nunits;
9076 : }
9077 833 : vec_perm_indices indices (sel, 2, nunits);
9078 : /* Don't use vect_gen_perm_mask_checked since can_vec_perm_const_p may
9079 : fail when vec_init is const vector. In that situation vec_perm is not
9080 : really needed. */
9081 833 : tree perm_mask_even
9082 833 : = vect_gen_perm_mask_any (vectype, indices);
9083 833 : vec_init = gimple_build (stmts, VEC_PERM_EXPR,
9084 : vectype,
9085 : vec_init, vec_neg,
9086 : perm_mask_even);
9087 833 : }
9088 833 : break;
9089 :
9090 78 : case vect_step_op_mul:
9091 78 : {
9092 : /* Use unsigned mult to avoid UD integer overflow. */
9093 78 : gcc_assert (nunits.is_constant (&const_nunits));
9094 78 : tree utype = unsigned_type_for (itype);
9095 78 : tree uvectype = build_vector_type (utype,
9096 78 : TYPE_VECTOR_SUBPARTS (vectype));
9097 78 : new_name = gimple_convert (stmts, utype, new_name);
9098 78 : vec_init = gimple_build_vector_from_val (stmts,
9099 : uvectype,
9100 : new_name);
9101 78 : tree_vector_builder elts (uvectype, const_nunits, 1);
9102 78 : tree elt_step = build_one_cst (utype);
9103 :
9104 78 : elts.quick_push (elt_step);
9105 670 : for (i = 1; i < const_nunits; i++)
9106 : {
9107 : /* Create: new_name_i = new_name + step_expr. */
9108 514 : elt_step = gimple_build (stmts, MULT_EXPR,
9109 : utype, elt_step, step_expr);
9110 514 : elts.quick_push (elt_step);
9111 : }
9112 : /* Create a vector from [new_name_0, new_name_1, ...,
9113 : new_name_nunits-1]. */
9114 78 : tree vec_mul = gimple_build_vector (stmts, &elts);
9115 78 : vec_init = gimple_build (stmts, MULT_EXPR, uvectype,
9116 : vec_init, vec_mul);
9117 78 : vec_init = gimple_convert (stmts, vectype, vec_init);
9118 78 : }
9119 78 : break;
9120 :
9121 0 : default:
9122 0 : gcc_unreachable ();
9123 : }
9124 :
9125 932 : return vec_init;
9126 : }
9127 :
9128 : /* Peel init_expr by skip_niter for induction_type. */
9129 : tree
9130 83 : vect_peel_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
9131 : tree skip_niters, tree step_expr,
9132 : enum vect_induction_op_type induction_type,
9133 : bool early_exit_p)
9134 : {
9135 83 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST || early_exit_p);
9136 83 : tree type = TREE_TYPE (init_expr);
9137 83 : unsigned prec = TYPE_PRECISION (type);
9138 83 : switch (induction_type)
9139 : {
9140 : /* neg inductions are typically not used for loop termination conditions but
9141 : are typically implemented as b = -b. That is every scalar iteration b is
9142 : negated. That means that for the initial value of b we will have to
9143 : determine whether the number of skipped iteration is a multiple of 2
9144 : because every 2 scalar iterations we are back at "b". */
9145 0 : case vect_step_op_neg:
9146 : /* For early exits the neg induction will always be the same value at the
9147 : start of the iteration. */
9148 0 : if (early_exit_p)
9149 : break;
9150 :
9151 0 : if (TREE_INT_CST_LOW (skip_niters) % 2)
9152 0 : init_expr = gimple_build (stmts, NEGATE_EXPR, type, init_expr);
9153 : /* else no change. */
9154 : break;
9155 :
9156 12 : case vect_step_op_shr:
9157 12 : case vect_step_op_shl:
9158 : /* When shift count >= precision, need to avoid UD.
9159 : In the original loop, there's no UD, and according to semantic,
9160 : init_expr should be 0 for lshr, ashl, and >>= (prec - 1) for ashr. */
9161 24 : if (!early_exit_p
9162 24 : && wi::mul (wi::to_widest (step_expr),
9163 36 : wi::to_widest (skip_niters)) >= prec)
9164 : {
9165 6 : if (induction_type == vect_step_op_shl
9166 6 : || TYPE_UNSIGNED (type))
9167 4 : init_expr = build_zero_cst (type);
9168 : else
9169 2 : init_expr = gimple_build (stmts, RSHIFT_EXPR, type,
9170 : init_expr,
9171 4 : wide_int_to_tree (type, prec - 1));
9172 : }
9173 : else
9174 : {
9175 : /* ??? For not constant skip_niters we'd have to treat the
9176 : overflow case. */
9177 6 : gcc_assert (!early_exit_p);
9178 6 : step_expr = fold_build2 (MULT_EXPR, TREE_TYPE (skip_niters),
9179 : skip_niters,
9180 : fold_convert (TREE_TYPE (skip_niters),
9181 : step_expr));
9182 8 : init_expr = fold_build2 ((induction_type == vect_step_op_shr
9183 : ? RSHIFT_EXPR : LSHIFT_EXPR),
9184 : type, init_expr, step_expr);
9185 6 : init_expr = force_gimple_operand (init_expr, stmts, false, NULL);
9186 : }
9187 : break;
9188 :
9189 71 : case vect_step_op_mul:
9190 71 : {
9191 : /* Due to UB we can't support vect_step_op_mul with early break for now.
9192 : so assert and block. */
9193 71 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST);
9194 71 : tree utype = unsigned_type_for (type);
9195 71 : init_expr = gimple_convert (stmts, utype, init_expr);
9196 71 : wide_int skipn = wi::to_wide (skip_niters);
9197 71 : wide_int begin = wi::to_wide (step_expr);
9198 71 : auto_mpz base, exp, mod, res;
9199 71 : wi::to_mpz (begin, base, TYPE_SIGN (type));
9200 71 : wi::to_mpz (skipn, exp, UNSIGNED);
9201 71 : mpz_ui_pow_ui (mod, 2, TYPE_PRECISION (type));
9202 71 : mpz_powm (res, base, exp, mod);
9203 71 : begin = wi::from_mpz (utype, res, true);
9204 71 : tree mult_expr = wide_int_to_tree (utype, begin);
9205 71 : init_expr = gimple_build (stmts, MULT_EXPR, utype,
9206 : init_expr, mult_expr);
9207 71 : init_expr = gimple_convert (stmts, type, init_expr);
9208 71 : }
9209 71 : break;
9210 :
9211 0 : default:
9212 0 : gcc_unreachable ();
9213 : }
9214 :
9215 83 : return init_expr;
9216 : }
9217 :
9218 : /* Create vector step for vectorized iv. */
9219 : static tree
9220 1221 : vect_create_nonlinear_iv_step (gimple_seq* stmts, tree step_expr,
9221 : poly_uint64 vf,
9222 : enum vect_induction_op_type induction_type)
9223 : {
9224 1221 : tree expr = build_int_cst (TREE_TYPE (step_expr), vf);
9225 1221 : tree new_name = NULL;
9226 : /* Step should be pow (step, vf) for mult induction. */
9227 1221 : if (induction_type == vect_step_op_mul)
9228 : {
9229 78 : gcc_assert (vf.is_constant ());
9230 78 : wide_int begin = wi::to_wide (step_expr);
9231 :
9232 592 : for (unsigned i = 0; i != vf.to_constant () - 1; i++)
9233 514 : begin = wi::mul (begin, wi::to_wide (step_expr));
9234 :
9235 78 : new_name = wide_int_to_tree (TREE_TYPE (step_expr), begin);
9236 78 : }
9237 1143 : else if (induction_type == vect_step_op_neg)
9238 : /* Do nothing. */
9239 : ;
9240 : else
9241 24 : new_name = gimple_build (stmts, MULT_EXPR, TREE_TYPE (step_expr),
9242 : expr, step_expr);
9243 1221 : return new_name;
9244 : }
9245 :
9246 : static tree
9247 1221 : vect_create_nonlinear_iv_vec_step (loop_vec_info loop_vinfo,
9248 : stmt_vec_info stmt_info,
9249 : tree new_name, tree vectype,
9250 : enum vect_induction_op_type induction_type)
9251 : {
9252 : /* No step is needed for neg induction. */
9253 1221 : if (induction_type == vect_step_op_neg)
9254 : return NULL;
9255 :
9256 102 : tree t = unshare_expr (new_name);
9257 102 : gcc_assert (CONSTANT_CLASS_P (new_name)
9258 : || TREE_CODE (new_name) == SSA_NAME);
9259 102 : tree new_vec = build_vector_from_val (vectype, t);
9260 102 : tree vec_step = vect_init_vector (loop_vinfo, stmt_info,
9261 : new_vec, vectype, NULL);
9262 102 : return vec_step;
9263 : }
9264 :
9265 : /* Update vectorized iv with vect_step, induc_def is init. */
9266 : static tree
9267 1427 : vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
9268 : tree induc_def, tree vec_step,
9269 : enum vect_induction_op_type induction_type)
9270 : {
9271 1427 : tree vec_def = induc_def;
9272 1427 : switch (induction_type)
9273 : {
9274 78 : case vect_step_op_mul:
9275 78 : {
9276 : /* Use unsigned mult to avoid UD integer overflow. */
9277 78 : tree uvectype = unsigned_type_for (vectype);
9278 78 : vec_def = gimple_convert (stmts, uvectype, vec_def);
9279 78 : vec_step = gimple_convert (stmts, uvectype, vec_step);
9280 78 : vec_def = gimple_build (stmts, MULT_EXPR, uvectype,
9281 : vec_def, vec_step);
9282 78 : vec_def = gimple_convert (stmts, vectype, vec_def);
9283 : }
9284 78 : break;
9285 :
9286 36 : case vect_step_op_shr:
9287 36 : vec_def = gimple_build (stmts, RSHIFT_EXPR, vectype,
9288 : vec_def, vec_step);
9289 36 : break;
9290 :
9291 6 : case vect_step_op_shl:
9292 6 : vec_def = gimple_build (stmts, LSHIFT_EXPR, vectype,
9293 : vec_def, vec_step);
9294 6 : break;
9295 : case vect_step_op_neg:
9296 : vec_def = induc_def;
9297 : /* Do nothing. */
9298 : break;
9299 0 : default:
9300 0 : gcc_unreachable ();
9301 : }
9302 :
9303 1427 : return vec_def;
9304 :
9305 : }
9306 :
9307 : /* Function vectorizable_nonlinear_induction
9308 :
9309 : Check if STMT_INFO performs an nonlinear induction computation that can be
9310 : vectorized. If VEC_STMT is also passed, vectorize the induction PHI: create
9311 : a vectorized phi to replace it, put it in VEC_STMT, and add it to the same
9312 : basic block.
9313 : Return true if STMT_INFO is vectorizable in this way. */
9314 :
9315 : static bool
9316 13483 : vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
9317 : stmt_vec_info stmt_info,
9318 : slp_tree slp_node,
9319 : stmt_vector_for_cost *cost_vec)
9320 : {
9321 13483 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9322 13483 : unsigned ncopies;
9323 13483 : bool nested_in_vect_loop = false;
9324 13483 : class loop *iv_loop;
9325 13483 : tree vec_def;
9326 13483 : edge pe = loop_preheader_edge (loop);
9327 13483 : basic_block new_bb;
9328 13483 : tree vec_init, vec_step;
9329 13483 : tree new_name;
9330 13483 : gimple *new_stmt;
9331 13483 : gphi *induction_phi;
9332 13483 : tree induc_def, vec_dest;
9333 13483 : tree init_expr, step_expr;
9334 13483 : tree niters_skip;
9335 13483 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9336 13483 : unsigned i;
9337 13483 : gimple_stmt_iterator si;
9338 :
9339 13483 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9340 :
9341 13483 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9342 13483 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9343 13483 : enum vect_induction_op_type induction_type
9344 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9345 :
9346 13483 : gcc_assert (induction_type > vect_step_op_add);
9347 :
9348 13483 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
9349 13483 : gcc_assert (ncopies >= 1);
9350 :
9351 : /* FORNOW. Only handle nonlinear induction in the same loop. */
9352 13483 : if (nested_in_vect_loop_p (loop, stmt_info))
9353 : {
9354 0 : if (dump_enabled_p ())
9355 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9356 : "nonlinear induction in nested loop.\n");
9357 : return false;
9358 : }
9359 :
9360 13483 : iv_loop = loop;
9361 13483 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9362 :
9363 : /* TODO: Support multi-lane SLP for nonlinear iv. There should be separate
9364 : vector iv update for each iv and a permutation to generate wanted
9365 : vector iv. */
9366 13483 : if (SLP_TREE_LANES (slp_node) > 1)
9367 : {
9368 0 : if (dump_enabled_p ())
9369 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9370 : "SLP induction not supported for nonlinear"
9371 : " induction.\n");
9372 : return false;
9373 : }
9374 :
9375 13483 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
9376 : {
9377 0 : if (dump_enabled_p ())
9378 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9379 : "floating point nonlinear induction vectorization"
9380 : " not supported.\n");
9381 : return false;
9382 : }
9383 :
9384 13483 : step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9385 13483 : init_expr = vect_phi_initial_value (phi);
9386 13483 : gcc_assert (step_expr != NULL_TREE && init_expr != NULL
9387 : && TREE_CODE (step_expr) == INTEGER_CST);
9388 : /* step_expr should be aligned with init_expr,
9389 : .i.e. uint64 a >> 1, step is int, but vector<uint64> shift is used. */
9390 13483 : step_expr = fold_convert (TREE_TYPE (vectype), step_expr);
9391 :
9392 13483 : if (TREE_CODE (init_expr) == INTEGER_CST)
9393 4141 : init_expr = fold_convert (TREE_TYPE (vectype), init_expr);
9394 9342 : else if (!tree_nop_conversion_p (TREE_TYPE (vectype), TREE_TYPE (init_expr)))
9395 : {
9396 : /* INIT_EXPR could be a bit_field, bail out for such case. */
9397 4 : if (dump_enabled_p ())
9398 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9399 : "nonlinear induction vectorization failed:"
9400 : " component type of vectype is not a nop conversion"
9401 : " from type of init_expr.\n");
9402 : return false;
9403 : }
9404 :
9405 13479 : switch (induction_type)
9406 : {
9407 3719 : case vect_step_op_neg:
9408 3719 : if (maybe_eq (TYPE_VECTOR_SUBPARTS (vectype), 1u))
9409 : return false;
9410 3557 : if (TREE_CODE (init_expr) != INTEGER_CST
9411 285 : && TREE_CODE (init_expr) != REAL_CST)
9412 : {
9413 : /* Check for backend support of NEGATE_EXPR and vec_perm. */
9414 285 : if (!directly_supported_p (NEGATE_EXPR, vectype))
9415 0 : return false;
9416 :
9417 : /* The encoding has 2 interleaved stepped patterns. */
9418 285 : vec_perm_builder sel (nunits, 2, 3);
9419 285 : machine_mode mode = TYPE_MODE (vectype);
9420 285 : sel.quick_grow (6);
9421 1425 : for (i = 0; i < 3; i++)
9422 : {
9423 855 : sel[i * 2] = i;
9424 855 : sel[i * 2 + 1] = i + nunits;
9425 : }
9426 285 : vec_perm_indices indices (sel, 2, nunits);
9427 285 : if (!can_vec_perm_const_p (mode, mode, indices))
9428 0 : return false;
9429 285 : }
9430 : break;
9431 :
9432 1072 : case vect_step_op_mul:
9433 1072 : {
9434 : /* Check for backend support of MULT_EXPR. */
9435 1072 : if (!directly_supported_p (MULT_EXPR, vectype))
9436 : return false;
9437 :
9438 : /* ?? How to construct vector step for variable number vector.
9439 : [ 1, step, pow (step, 2), pow (step, 4), .. ]. */
9440 : if (!vf.is_constant ())
9441 : return false;
9442 : }
9443 : break;
9444 :
9445 8286 : case vect_step_op_shr:
9446 : /* Check for backend support of RSHIFT_EXPR. */
9447 8286 : if (!directly_supported_p (RSHIFT_EXPR, vectype, optab_vector))
9448 : return false;
9449 :
9450 : /* Don't shift more than type precision to avoid UB. */
9451 98 : if (!tree_fits_uhwi_p (step_expr)
9452 98 : || maybe_ge (vf * tree_to_uhwi (step_expr),
9453 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9454 : return false;
9455 : break;
9456 :
9457 402 : case vect_step_op_shl:
9458 : /* Check for backend support of RSHIFT_EXPR. */
9459 402 : if (!directly_supported_p (LSHIFT_EXPR, vectype, optab_vector))
9460 : return false;
9461 :
9462 : /* Don't shift more than type precision to avoid UB. */
9463 42 : if (!tree_fits_uhwi_p (step_expr)
9464 42 : || maybe_ge (vf * tree_to_uhwi (step_expr),
9465 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9466 : return false;
9467 : break;
9468 :
9469 0 : default:
9470 0 : gcc_unreachable ();
9471 : }
9472 :
9473 4491 : if (cost_vec) /* transformation not required. */
9474 : {
9475 3559 : unsigned inside_cost = 0, prologue_cost = 0;
9476 : /* loop cost for vec_loop. Neg induction doesn't have any
9477 : inside_cost. */
9478 3559 : inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
9479 : slp_node, 0, vect_body);
9480 :
9481 : /* loop cost for vec_loop. Neg induction doesn't have any
9482 : inside_cost. */
9483 3559 : if (induction_type == vect_step_op_neg)
9484 2724 : inside_cost = 0;
9485 :
9486 : /* prologue cost for vec_init and vec_step. */
9487 3559 : prologue_cost = record_stmt_cost (cost_vec, 2, scalar_to_vec,
9488 : slp_node, 0, vect_prologue);
9489 :
9490 3559 : if (dump_enabled_p ())
9491 133 : dump_printf_loc (MSG_NOTE, vect_location,
9492 : "vect_model_induction_cost: inside_cost = %d, "
9493 : "prologue_cost = %d. \n", inside_cost,
9494 : prologue_cost);
9495 :
9496 3559 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9497 3559 : DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
9498 3559 : return true;
9499 : }
9500 :
9501 : /* Transform. */
9502 :
9503 : /* Compute a vector variable, initialized with the first VF values of
9504 : the induction variable. E.g., for an iv with IV_PHI='X' and
9505 : evolution S, for a vector of 4 units, we want to compute:
9506 : [X, X + S, X + 2*S, X + 3*S]. */
9507 :
9508 932 : if (dump_enabled_p ())
9509 43 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9510 :
9511 932 : pe = loop_preheader_edge (iv_loop);
9512 : /* Find the first insertion point in the BB. */
9513 932 : basic_block bb = gimple_bb (phi);
9514 932 : si = gsi_after_labels (bb);
9515 :
9516 932 : gimple_seq stmts = NULL;
9517 :
9518 932 : niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
9519 : /* If we are using the loop mask to "peel" for alignment then we need
9520 : to adjust the start value here. */
9521 932 : if (niters_skip != NULL_TREE)
9522 0 : init_expr = vect_peel_nonlinear_iv_init (&stmts, init_expr, niters_skip,
9523 : step_expr, induction_type, false);
9524 :
9525 932 : vec_init = vect_create_nonlinear_iv_init (&stmts, init_expr,
9526 : step_expr, nunits, vectype,
9527 : induction_type);
9528 932 : if (stmts)
9529 : {
9530 167 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9531 167 : gcc_assert (!new_bb);
9532 : }
9533 :
9534 932 : stmts = NULL;
9535 932 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9536 : vf, induction_type);
9537 932 : if (stmts)
9538 : {
9539 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9540 0 : gcc_assert (!new_bb);
9541 : }
9542 :
9543 932 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9544 : new_name, vectype,
9545 : induction_type);
9546 : /* Create the following def-use cycle:
9547 : loop prolog:
9548 : vec_init = ...
9549 : vec_step = ...
9550 : loop:
9551 : vec_iv = PHI <vec_init, vec_loop>
9552 : ...
9553 : STMT
9554 : ...
9555 : vec_loop = vec_iv + vec_step; */
9556 :
9557 : /* Create the induction-phi that defines the induction-operand. */
9558 932 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
9559 932 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
9560 932 : induc_def = PHI_RESULT (induction_phi);
9561 :
9562 : /* Create the iv update inside the loop. */
9563 932 : stmts = NULL;
9564 932 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9565 : induc_def, vec_step,
9566 : induction_type);
9567 :
9568 932 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9569 932 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9570 :
9571 : /* Set the arguments of the phi node: */
9572 932 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
9573 932 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
9574 : UNKNOWN_LOCATION);
9575 :
9576 932 : slp_node->push_vec_def (induction_phi);
9577 :
9578 : /* In case that vectorization factor (VF) is bigger than the number
9579 : of elements that we can fit in a vectype (nunits), we have to generate
9580 : more than one vector stmt - i.e - we need to "unroll" the
9581 : vector stmt by a factor VF/nunits. For more details see documentation
9582 : in vectorizable_operation. */
9583 :
9584 932 : if (ncopies > 1)
9585 : {
9586 289 : stmts = NULL;
9587 : /* FORNOW. This restriction should be relaxed. */
9588 289 : gcc_assert (!nested_in_vect_loop);
9589 :
9590 289 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9591 : nunits, induction_type);
9592 :
9593 289 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9594 : new_name, vectype,
9595 : induction_type);
9596 289 : vec_def = induc_def;
9597 1073 : for (i = 1; i < ncopies; i++)
9598 : {
9599 : /* vec_i = vec_prev + vec_step. */
9600 495 : stmts = NULL;
9601 495 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9602 : vec_def, vec_step,
9603 : induction_type);
9604 495 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9605 495 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9606 495 : slp_node->push_vec_def (new_stmt);
9607 : }
9608 : }
9609 :
9610 932 : if (dump_enabled_p ())
9611 86 : dump_printf_loc (MSG_NOTE, vect_location,
9612 : "transform induction: created def-use cycle: %G%G",
9613 43 : (gimple *) induction_phi, SSA_NAME_DEF_STMT (vec_def));
9614 :
9615 : return true;
9616 : }
9617 :
9618 : /* Return true if the scalar initial values and steps of the SLP induction
9619 : lanes allow the first CANDIDATE_NIVS IVs to be reused circularly for the
9620 : remaining lanes. */
9621 : static bool
9622 26 : vect_slp_induction_reuse_p (tree *steps, tree *inits, unsigned group_size,
9623 : unsigned HOST_WIDE_INT const_nunits,
9624 : unsigned candidate_nivs, unsigned nivs)
9625 : {
9626 26 : gcc_assert (candidate_nivs > 0);
9627 26 : gcc_assert (candidate_nivs < nivs);
9628 :
9629 : /* This function compares only STEPS and INITS, so all checked lanes must
9630 : precede the first wrap of the SLP group. */
9631 26 : gcc_assert (nivs * const_nunits <= group_size);
9632 :
9633 62 : for (unsigned ivn = candidate_nivs; ivn < nivs; ++ivn)
9634 : {
9635 42 : unsigned reuse_ivn = ivn % candidate_nivs;
9636 172 : for (unsigned HOST_WIDE_INT eltn = 0; eltn < const_nunits; ++eltn)
9637 : {
9638 136 : unsigned HOST_WIDE_INT elt = ivn * const_nunits + eltn;
9639 136 : unsigned HOST_WIDE_INT reused_elt
9640 136 : = reuse_ivn * const_nunits + eltn;
9641 :
9642 136 : if (!operand_equal_p (steps[elt], steps[reused_elt], 0)
9643 136 : || !operand_equal_p (inits[elt], inits[reused_elt], 0))
9644 : return false;
9645 : }
9646 : }
9647 :
9648 : return true;
9649 : }
9650 :
9651 : /* Function vectorizable_induction
9652 :
9653 : Check if STMT_INFO performs an induction computation that can be vectorized.
9654 : If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
9655 : phi to replace it, put it in VEC_STMT, and add it to the same basic block.
9656 : Return true if STMT_INFO is vectorizable in this way. */
9657 :
9658 : bool
9659 351732 : vectorizable_induction (loop_vec_info loop_vinfo,
9660 : stmt_vec_info stmt_info,
9661 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
9662 : {
9663 351732 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9664 351732 : bool nested_in_vect_loop = false;
9665 351732 : class loop *iv_loop;
9666 351732 : tree vec_def;
9667 351732 : edge pe = loop_preheader_edge (loop);
9668 351732 : basic_block new_bb;
9669 351732 : tree vec_init = NULL_TREE, vec_step, t;
9670 351732 : tree new_name;
9671 351732 : gphi *induction_phi;
9672 351732 : tree induc_def, vec_dest;
9673 351732 : unsigned i;
9674 351732 : tree index_vectype = NULL_TREE;
9675 351732 : gimple_stmt_iterator si;
9676 351732 : enum vect_induction_op_type induction_type
9677 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9678 :
9679 351732 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9680 189615 : if (!phi)
9681 : return false;
9682 :
9683 189615 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
9684 : return false;
9685 :
9686 : /* Make sure it was recognized as induction computation. */
9687 189615 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
9688 : return false;
9689 :
9690 746823 : for (auto phi_info : SLP_TREE_SCALAR_STMTS (slp_node))
9691 190395 : if (phi_info)
9692 : {
9693 190395 : gcc_assert (STMT_VINFO_DEF_TYPE (phi_info) == vect_induction_def);
9694 190395 : if (STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (phi_info) != induction_type)
9695 : return false;
9696 : }
9697 :
9698 : /* Handle nonlinear induction in a separate place. */
9699 185474 : if (induction_type != vect_step_op_add)
9700 13483 : return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
9701 13483 : slp_node, cost_vec);
9702 :
9703 171991 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9704 171991 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9705 :
9706 : /* FORNOW. These restrictions should be relaxed. */
9707 171991 : if (nested_in_vect_loop_p (loop, stmt_info))
9708 : {
9709 780 : imm_use_iterator imm_iter;
9710 780 : use_operand_p use_p;
9711 780 : gimple *exit_phi;
9712 780 : edge latch_e;
9713 780 : tree loop_arg;
9714 :
9715 780 : exit_phi = NULL;
9716 780 : latch_e = loop_latch_edge (loop->inner);
9717 780 : loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
9718 1597 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
9719 : {
9720 843 : gimple *use_stmt = USE_STMT (use_p);
9721 843 : if (is_gimple_debug (use_stmt))
9722 36 : continue;
9723 :
9724 807 : if (!flow_bb_inside_loop_p (loop->inner, gimple_bb (use_stmt)))
9725 : {
9726 : exit_phi = use_stmt;
9727 : break;
9728 : }
9729 780 : }
9730 780 : if (exit_phi)
9731 : {
9732 26 : stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi);
9733 26 : if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
9734 10 : && !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
9735 : {
9736 16 : if (dump_enabled_p ())
9737 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9738 : "inner-loop induction only used outside "
9739 : "of the outer vectorized loop.\n");
9740 16 : return false;
9741 : }
9742 : }
9743 :
9744 764 : nested_in_vect_loop = true;
9745 764 : iv_loop = loop->inner;
9746 : }
9747 : else
9748 : iv_loop = loop;
9749 171975 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9750 :
9751 171975 : if (!nunits.is_constant () && SLP_TREE_LANES (slp_node) != 1)
9752 : {
9753 : /* The current SLP code creates the step value element-by-element. */
9754 : if (dump_enabled_p ())
9755 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9756 : "SLP induction not supported for variable-length"
9757 : " vectors.\n");
9758 : return false;
9759 : }
9760 :
9761 171975 : if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
9762 : {
9763 12 : if (dump_enabled_p ())
9764 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9765 : "floating point induction vectorization disabled\n");
9766 : return false;
9767 : }
9768 :
9769 171963 : tree step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9770 171963 : gcc_assert (step_expr != NULL_TREE);
9771 343902 : if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
9772 343801 : && !type_has_mode_precision_p (TREE_TYPE (step_expr)))
9773 : {
9774 12 : if (dump_enabled_p ())
9775 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9776 : "bit-precision induction vectorization not "
9777 : "supported.\n");
9778 : return false;
9779 : }
9780 171951 : tree stept = TREE_TYPE (step_expr);
9781 171951 : tree step_vectype
9782 171951 : = (!INTEGRAL_TYPE_P (stept) ? vectype
9783 171850 : : signed_or_unsigned_type_for (TYPE_UNSIGNED (stept), vectype));
9784 171951 : stept = TREE_TYPE (step_vectype);
9785 :
9786 : /* Check for target support of the vectorized arithmetic used here. */
9787 171951 : if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
9788 171951 : || !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default))
9789 : return false;
9790 140251 : if (!nunits.is_constant ()
9791 140251 : || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9792 : {
9793 0 : if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default))
9794 : return false;
9795 : /* FLOAT_EXPR when computing VEC_INIT for float inductions. */
9796 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9797 : {
9798 0 : tree index_type = build_nonstandard_integer_type
9799 0 : (GET_MODE_BITSIZE (SCALAR_TYPE_MODE (stept)), 1);
9800 :
9801 0 : index_vectype = build_vector_type (index_type, nunits);
9802 0 : if (!can_float_p (TYPE_MODE (step_vectype),
9803 0 : TYPE_MODE (index_vectype), 1))
9804 : return false;
9805 : }
9806 : }
9807 :
9808 140251 : unsigned nvects = vect_get_num_copies (loop_vinfo, slp_node);
9809 140251 : if (cost_vec) /* transformation not required. */
9810 : {
9811 372516 : unsigned inside_cost = 0, prologue_cost = 0;
9812 : /* We eventually need to set a vector type on invariant
9813 : arguments. */
9814 : unsigned j;
9815 : slp_tree child;
9816 372516 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
9817 248344 : if (!vect_maybe_update_slp_op_vectype
9818 248344 : (child, SLP_TREE_VECTYPE (slp_node)))
9819 : {
9820 0 : if (dump_enabled_p ())
9821 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9822 : "incompatible vector types for "
9823 : "invariants\n");
9824 : return false;
9825 : }
9826 : /* loop cost for vec_loop. */
9827 124172 : inside_cost = record_stmt_cost (cost_vec, nvects,
9828 : vector_stmt, slp_node, 0, vect_body);
9829 : /* prologue cost for vec_init (if not nested) and step. */
9830 124172 : prologue_cost = record_stmt_cost (cost_vec, 1 + !nested_in_vect_loop,
9831 : scalar_to_vec,
9832 : slp_node, 0, vect_prologue);
9833 124172 : if (dump_enabled_p ())
9834 4373 : dump_printf_loc (MSG_NOTE, vect_location,
9835 : "vect_model_induction_cost: inside_cost = %d, "
9836 : "prologue_cost = %d .\n", inside_cost,
9837 : prologue_cost);
9838 :
9839 124172 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9840 124172 : DUMP_VECT_SCOPE ("vectorizable_induction");
9841 124172 : return true;
9842 : }
9843 :
9844 : /* Transform. */
9845 :
9846 : /* Compute a vector variable, initialized with the first VF values of
9847 : the induction variable. E.g., for an iv with IV_PHI='X' and
9848 : evolution S, for a vector of 4 units, we want to compute:
9849 : [X, X + S, X + 2*S, X + 3*S]. */
9850 :
9851 16079 : if (dump_enabled_p ())
9852 2926 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9853 :
9854 16079 : pe = loop_preheader_edge (iv_loop);
9855 : /* Find the first insertion point in the BB. */
9856 16079 : basic_block bb = gimple_bb (phi);
9857 16079 : si = gsi_after_labels (bb);
9858 :
9859 : /* For SLP induction we have to generate several IVs as for example
9860 : with group size 3 we need
9861 : [i0, i1, i2, i0 + S0] [i1 + S1, i2 + S2, i0 + 2*S0, i1 + 2*S1]
9862 : [i2 + 2*S2, i0 + 3*S0, i1 + 3*S1, i2 + 3*S2]. */
9863 16079 : gimple_stmt_iterator incr_si;
9864 16079 : bool insert_after;
9865 16079 : standard_iv_increment_position (iv_loop, &incr_si, &insert_after);
9866 :
9867 : /* The initial values are vectorized, but any lanes > group_size
9868 : need adjustment. */
9869 16079 : slp_tree init_node
9870 16079 : = SLP_TREE_CHILDREN (slp_node)[pe->dest_idx];
9871 :
9872 : /* Gather steps. Since we do not vectorize inductions as
9873 : cycles we have to reconstruct the step from SCEV data. */
9874 16079 : unsigned group_size = SLP_TREE_LANES (slp_node);
9875 16079 : tree *steps = XALLOCAVEC (tree, group_size);
9876 16079 : tree *inits = XALLOCAVEC (tree, group_size);
9877 16079 : stmt_vec_info phi_info;
9878 49663 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, phi_info)
9879 : {
9880 17505 : steps[i] = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
9881 17505 : if (!init_node)
9882 17277 : inits[i] = gimple_phi_arg_def (as_a<gphi *> (phi_info->stmt),
9883 : pe->dest_idx);
9884 : }
9885 :
9886 : /* Now generate the IVs. */
9887 32158 : gcc_assert (multiple_p (nunits * nvects, group_size));
9888 16079 : unsigned nivs;
9889 16079 : unsigned HOST_WIDE_INT const_nunits;
9890 16079 : if (nested_in_vect_loop)
9891 : nivs = nvects;
9892 15872 : else if (nunits.is_constant (&const_nunits)
9893 15872 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9894 : {
9895 15872 : gcc_assert (!init_node);
9896 : /* Compute the number of distinct IVs we need. We can reduce the
9897 : number when later vector chunks are equal to earlier chunks. */
9898 15872 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
9899 15872 : unsigned group_sizep = group_size;
9900 15872 : if (group_sizep % const_nunits == 0)
9901 : {
9902 124 : group_sizep = group_sizep / const_nunits;
9903 124 : unsigned candidate_nivs
9904 124 : = least_common_multiple (group_sizep, const_nunits) / const_nunits;
9905 124 : if (candidate_nivs < nivs
9906 124 : && vect_slp_induction_reuse_p (steps, inits, group_size,
9907 : const_nunits, candidate_nivs, nivs))
9908 : {
9909 20 : if (dump_enabled_p ())
9910 16 : dump_printf_loc (MSG_NOTE, vect_location,
9911 : "reusing %u SLP induction IVs for %u "
9912 : "vector chunks\n",
9913 : candidate_nivs, nivs);
9914 : nivs = candidate_nivs;
9915 : }
9916 : }
9917 : }
9918 : else
9919 : {
9920 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
9921 : nivs = 1;
9922 : }
9923 16079 : gimple_seq init_stmts = NULL;
9924 16079 : gimple_seq lupdate_mul_stmts = NULL;
9925 16079 : tree lupdate_mul = NULL_TREE;
9926 16079 : if (!nested_in_vect_loop)
9927 : {
9928 15872 : if (nunits.is_constant (&const_nunits)
9929 15872 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9930 : {
9931 : /* The number of iterations covered in one vector iteration. */
9932 15872 : unsigned lup_mul = (nvects * const_nunits) / group_size;
9933 15872 : lupdate_mul
9934 15872 : = build_vector_from_val (step_vectype,
9935 15872 : SCALAR_FLOAT_TYPE_P (stept)
9936 28 : ? build_real_from_wide (stept, lup_mul,
9937 : UNSIGNED)
9938 31716 : : build_int_cstu (stept, lup_mul));
9939 : }
9940 : else
9941 : {
9942 0 : gimple_seq *update_stmts
9943 : = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
9944 : ? &init_stmts
9945 : : &lupdate_mul_stmts;
9946 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9947 : {
9948 0 : tree increment
9949 0 : = gimple_convert (update_stmts, integer_type_node,
9950 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9951 0 : lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept,
9952 : increment);
9953 : }
9954 : else
9955 0 : lupdate_mul = gimple_convert (update_stmts, stept,
9956 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9957 0 : lupdate_mul = gimple_build_vector_from_val (update_stmts,
9958 : step_vectype,
9959 : lupdate_mul);
9960 : }
9961 : }
9962 16079 : tree peel_mul = NULL_TREE;
9963 16079 : if (LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo))
9964 : {
9965 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9966 0 : peel_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept,
9967 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9968 : else
9969 0 : peel_mul = gimple_convert (&init_stmts, stept,
9970 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9971 0 : peel_mul = gimple_build_vector_from_val (&init_stmts,
9972 : step_vectype, peel_mul);
9973 : }
9974 16079 : tree step_mul = NULL_TREE;
9975 16079 : unsigned ivn;
9976 16079 : auto_vec<tree> vec_steps;
9977 32746 : for (ivn = 0; ivn < nivs; ++ivn)
9978 : {
9979 16667 : gimple_seq stmts = NULL;
9980 16667 : bool invariant = true;
9981 16667 : if (nunits.is_constant (&const_nunits)
9982 16667 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9983 : {
9984 16667 : tree_vector_builder step_elts (step_vectype, const_nunits, 1);
9985 16667 : tree_vector_builder init_elts (vectype, const_nunits, 1);
9986 16667 : tree_vector_builder mul_elts (step_vectype, const_nunits, 1);
9987 122954 : for (unsigned eltn = 0; eltn < const_nunits; ++eltn)
9988 : {
9989 : /* The scalar steps of the IVs. */
9990 89620 : tree elt = steps[(ivn*const_nunits + eltn) % group_size];
9991 89620 : elt = gimple_convert (&init_stmts, TREE_TYPE (step_vectype), elt);
9992 89620 : step_elts.quick_push (elt);
9993 89620 : if (!init_node)
9994 : {
9995 : /* The scalar inits of the IVs if not vectorized. */
9996 88414 : elt = inits[(ivn*const_nunits + eltn) % group_size];
9997 88414 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
9998 88414 : TREE_TYPE (elt)))
9999 324 : elt = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
10000 324 : TREE_TYPE (vectype), elt);
10001 88414 : init_elts.quick_push (elt);
10002 : }
10003 : /* The number of steps to add to the initial values. */
10004 89620 : unsigned mul_elt = (ivn*const_nunits + eltn) / group_size;
10005 179240 : mul_elts.quick_push (SCALAR_FLOAT_TYPE_P (stept)
10006 179138 : ? build_real_from_wide (stept, mul_elt,
10007 : UNSIGNED)
10008 179138 : : build_int_cstu (stept, mul_elt));
10009 : }
10010 16667 : vec_step = gimple_build_vector (&init_stmts, &step_elts);
10011 16667 : step_mul = gimple_build_vector (&init_stmts, &mul_elts);
10012 16667 : if (!init_node)
10013 16426 : vec_init = gimple_build_vector (&init_stmts, &init_elts);
10014 16667 : }
10015 : else
10016 : {
10017 0 : tree step = gimple_convert (&init_stmts, stept, steps[0]);
10018 0 : if (init_node)
10019 : ;
10020 0 : else if (INTEGRAL_TYPE_P (stept))
10021 : {
10022 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
10023 : /* Build the initial value directly as a VEC_SERIES_EXPR. */
10024 0 : vec_init = gimple_build (&init_stmts, VEC_SERIES_EXPR,
10025 : step_vectype, new_name, step);
10026 0 : if (!useless_type_conversion_p (vectype, step_vectype))
10027 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
10028 : vectype, vec_init);
10029 : }
10030 : else
10031 : {
10032 : /* Build:
10033 : [base, base, base, ...]
10034 : + (vectype) [0, 1, 2, ...] * [step, step, step, ...]. */
10035 0 : gcc_assert (SCALAR_FLOAT_TYPE_P (stept));
10036 0 : gcc_assert (flag_associative_math);
10037 0 : gcc_assert (index_vectype != NULL_TREE);
10038 :
10039 0 : tree index = build_index_vector (index_vectype, 0, 1);
10040 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
10041 0 : tree base_vec = gimple_build_vector_from_val (&init_stmts,
10042 : step_vectype,
10043 : new_name);
10044 0 : tree step_vec = gimple_build_vector_from_val (&init_stmts,
10045 : step_vectype,
10046 : step);
10047 0 : vec_init = gimple_build (&init_stmts, FLOAT_EXPR,
10048 : step_vectype, index);
10049 0 : vec_init = gimple_build (&init_stmts, MULT_EXPR,
10050 : step_vectype, vec_init, step_vec);
10051 0 : vec_init = gimple_build (&init_stmts, PLUS_EXPR,
10052 : step_vectype, vec_init, base_vec);
10053 0 : if (!useless_type_conversion_p (vectype, step_vectype))
10054 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
10055 : vectype, vec_init);
10056 : }
10057 : /* iv_loop is nested in the loop to be vectorized. Generate:
10058 : vec_step = [S, S, S, S] */
10059 0 : t = unshare_expr (step);
10060 0 : gcc_assert (CONSTANT_CLASS_P (t)
10061 : || TREE_CODE (t) == SSA_NAME);
10062 0 : vec_step = gimple_build_vector_from_val (&init_stmts,
10063 : step_vectype, t);
10064 : }
10065 16667 : vec_steps.safe_push (vec_step);
10066 16667 : if (peel_mul)
10067 : {
10068 0 : if (!step_mul)
10069 : {
10070 0 : gcc_assert (!nunits.is_constant ());
10071 : step_mul = gimple_build (&init_stmts,
10072 : MINUS_EXPR, step_vectype,
10073 : build_zero_cst (step_vectype), peel_mul);
10074 : }
10075 : else
10076 0 : step_mul = gimple_build (&init_stmts,
10077 : MINUS_EXPR, step_vectype,
10078 : step_mul, peel_mul);
10079 : }
10080 :
10081 : /* Create the induction-phi that defines the induction-operand. */
10082 16667 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var,
10083 : "vec_iv_");
10084 16667 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
10085 16667 : induc_def = PHI_RESULT (induction_phi);
10086 :
10087 : /* Create the iv update inside the loop */
10088 16667 : tree up = vec_step;
10089 16667 : if (lupdate_mul)
10090 : {
10091 16426 : if (lupdate_mul_stmts)
10092 0 : gimple_seq_add_seq (&stmts, lupdate_mul_stmts);
10093 16426 : up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step,
10094 : lupdate_mul);
10095 : }
10096 16667 : vec_def = gimple_convert (&stmts, step_vectype, induc_def);
10097 16667 : vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
10098 16667 : vec_def = gimple_convert (&stmts, vectype, vec_def);
10099 16667 : insert_iv_increment (&incr_si, insert_after, stmts);
10100 16667 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
10101 : UNKNOWN_LOCATION);
10102 :
10103 16667 : if (init_node)
10104 241 : vec_init = vect_get_slp_vect_def (init_node, ivn);
10105 16667 : if (!nested_in_vect_loop
10106 16667 : && step_mul
10107 16667 : && !integer_zerop (step_mul))
10108 : {
10109 15956 : gcc_assert (invariant);
10110 15956 : vec_def = gimple_convert (&init_stmts, step_vectype, vec_init);
10111 15956 : up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
10112 : vec_step, step_mul);
10113 15956 : vec_def = gimple_build (&init_stmts, PLUS_EXPR, step_vectype,
10114 : vec_def, up);
10115 15956 : vec_init = gimple_convert (&init_stmts, vectype, vec_def);
10116 : }
10117 :
10118 : /* Set the arguments of the phi node: */
10119 16667 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
10120 :
10121 16667 : slp_node->push_vec_def (induction_phi);
10122 : }
10123 16079 : if (!nested_in_vect_loop)
10124 : {
10125 : /* Fill up to the number of vectors we need for the whole group. */
10126 15872 : if (nunits.is_constant (&const_nunits)
10127 15872 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10128 15872 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
10129 : else
10130 : nivs = 1;
10131 15872 : vec_steps.reserve (nivs-ivn);
10132 15872 : unsigned generated_nivs = ivn;
10133 15872 : gcc_assert (generated_nivs > 0);
10134 15908 : for (; ivn < nivs; ++ivn)
10135 : {
10136 36 : unsigned reuse_ivn = ivn % generated_nivs;
10137 36 : slp_node->push_vec_def (SLP_TREE_VEC_DEFS (slp_node)[reuse_ivn]);
10138 36 : vec_steps.quick_push (vec_steps[reuse_ivn]);
10139 : }
10140 : }
10141 :
10142 : /* Re-use IVs when we can. We are generating further vector
10143 : stmts by adding VF' * stride to the IVs generated above. */
10144 16079 : if (ivn < nvects)
10145 : {
10146 3550 : if (nunits.is_constant (&const_nunits)
10147 3550 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10148 : {
10149 3550 : unsigned vfp = (least_common_multiple (group_size, const_nunits)
10150 3550 : / group_size);
10151 3550 : lupdate_mul
10152 3550 : = build_vector_from_val (step_vectype,
10153 3550 : SCALAR_FLOAT_TYPE_P (stept)
10154 8 : ? build_real_from_wide (stept,
10155 8 : vfp, UNSIGNED)
10156 7092 : : build_int_cstu (stept, vfp));
10157 : }
10158 : else
10159 : {
10160 0 : if (SCALAR_FLOAT_TYPE_P (stept))
10161 : {
10162 0 : tree tem = build_int_cst (integer_type_node, nunits);
10163 0 : lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem);
10164 : }
10165 : else
10166 0 : lupdate_mul = build_int_cst (stept, nunits);
10167 0 : lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype,
10168 : lupdate_mul);
10169 : }
10170 11498 : for (; ivn < nvects; ++ivn)
10171 : {
10172 7948 : gimple *iv
10173 7948 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node)[ivn - nivs]);
10174 7948 : tree def = gimple_get_lhs (iv);
10175 7948 : if (ivn < 2*nivs)
10176 3648 : vec_steps[ivn - nivs]
10177 3648 : = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
10178 3648 : vec_steps[ivn - nivs], lupdate_mul);
10179 7948 : gimple_seq stmts = NULL;
10180 7948 : def = gimple_convert (&stmts, step_vectype, def);
10181 23844 : def = gimple_build (&stmts, PLUS_EXPR, step_vectype,
10182 7948 : def, vec_steps[ivn % nivs]);
10183 7948 : def = gimple_convert (&stmts, vectype, def);
10184 7948 : if (gimple_code (iv) == GIMPLE_PHI)
10185 3648 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10186 : else
10187 : {
10188 4300 : gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
10189 4300 : gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING);
10190 : }
10191 7948 : slp_node->push_vec_def (def);
10192 : }
10193 : }
10194 :
10195 16079 : new_bb = gsi_insert_seq_on_edge_immediate (pe, init_stmts);
10196 16079 : gcc_assert (!new_bb);
10197 :
10198 16079 : return true;
10199 16079 : }
10200 :
10201 : /* Function vectorizable_live_operation_1.
10202 :
10203 : helper function for vectorizable_live_operation. */
10204 :
10205 : static tree
10206 2799 : vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
10207 : tree vectype, slp_tree slp_node,
10208 : tree bitsize, tree bitstart, tree vec_lhs,
10209 : tree lhs_type, gimple_stmt_iterator *exit_gsi)
10210 : {
10211 2799 : gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10212 :
10213 2799 : tree vec_lhs_phi = copy_ssa_name (vec_lhs);
10214 2799 : gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
10215 8399 : for (unsigned i = 0; i < gimple_phi_num_args (phi); i++)
10216 2801 : SET_PHI_ARG_DEF (phi, i, vec_lhs);
10217 :
10218 2799 : gimple_seq stmts = NULL;
10219 2799 : tree new_tree;
10220 :
10221 : /* If bitstart is 0 then we can use a BIT_FIELD_REF */
10222 2799 : if (integer_zerop (bitstart))
10223 : {
10224 212 : tree scalar_res = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
10225 : vec_lhs_phi, bitsize, bitstart);
10226 :
10227 : /* Convert the extracted vector element to the scalar type. */
10228 212 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10229 : }
10230 2587 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10231 : {
10232 : /* Emit:
10233 :
10234 : SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>
10235 :
10236 : where VEC_LHS is the vectorized live-out result, LEN is the length of
10237 : the vector, BIAS is the load-store bias. The bias should not be used
10238 : at all since we are not using load/store operations, but LEN will be
10239 : REALLEN + BIAS, so subtract it to get to the correct position. */
10240 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10241 0 : gimple_seq tem = NULL;
10242 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10243 0 : tree len = vect_get_loop_len (loop_vinfo, &gsi,
10244 : &LOOP_VINFO_LENS (loop_vinfo),
10245 : 1, vectype, 0, 1, false);
10246 0 : gimple_seq_add_seq (&stmts, tem);
10247 :
10248 : /* LAST_INDEX = LEN - 1. */
10249 0 : tree last_index = gimple_build (&stmts, MINUS_EXPR, TREE_TYPE (len),
10250 0 : len, build_one_cst (TREE_TYPE (len)));
10251 :
10252 : /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>. */
10253 0 : tree scalar_res
10254 0 : = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
10255 : vec_lhs_phi, last_index);
10256 :
10257 : /* Convert the extracted vector element to the scalar type. */
10258 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10259 : }
10260 2587 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
10261 : {
10262 : /* Emit:
10263 :
10264 : SCALAR_RES = EXTRACT_LAST_ACTIVE <VEC_LHS, MASK>
10265 :
10266 : where VEC_LHS is the vectorized live-out result and MASK is
10267 : the loop mask for the final iteration. */
10268 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10269 0 : tree scalar_type = TREE_TYPE (vectype);
10270 0 : gimple_seq tem = NULL;
10271 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10272 0 : tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
10273 : &LOOP_VINFO_MASKS (loop_vinfo),
10274 : 1, vectype, 0);
10275 0 : tree scalar_res;
10276 0 : gimple_seq_add_seq (&stmts, tem);
10277 :
10278 0 : scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST_ACTIVE, scalar_type,
10279 : mask, vec_lhs_phi);
10280 :
10281 : /* Convert the extracted vector element to the scalar type. */
10282 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10283 : }
10284 : else
10285 : {
10286 2587 : tree bftype = TREE_TYPE (vectype);
10287 2587 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10288 85 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10289 2587 : new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart);
10290 2587 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10291 : &stmts, true, NULL_TREE);
10292 : }
10293 :
10294 2799 : *exit_gsi = gsi_after_labels (exit_bb);
10295 2799 : if (stmts)
10296 2799 : gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT);
10297 :
10298 2799 : return new_tree;
10299 : }
10300 :
10301 : /* Function vectorizable_live_operation.
10302 :
10303 : STMT_INFO computes a value that is used outside the loop. Check if
10304 : it can be supported. */
10305 :
10306 : bool
10307 287949 : vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
10308 : slp_tree slp_node, slp_instance slp_node_instance,
10309 : int slp_index, bool vec_stmt_p,
10310 : stmt_vector_for_cost *cost_vec)
10311 : {
10312 287949 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
10313 287949 : imm_use_iterator imm_iter;
10314 287949 : tree lhs, lhs_type, bitsize;
10315 287949 : tree vectype = SLP_TREE_VECTYPE (slp_node);
10316 287949 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
10317 287949 : gimple *use_stmt;
10318 287949 : use_operand_p use_p;
10319 287949 : auto_vec<tree> vec_oprnds;
10320 287949 : int vec_entry = 0;
10321 287949 : poly_uint64 vec_index = 0;
10322 :
10323 287949 : gcc_assert (STMT_VINFO_LIVE_P (stmt_info)
10324 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10325 :
10326 : /* If a stmt of a reduction is live, vectorize it via
10327 : vect_create_epilog_for_reduction. vectorizable_reduction assessed
10328 : validity so just trigger the transform here. */
10329 287949 : if (vect_is_reduction (slp_node))
10330 : {
10331 84725 : if (!vec_stmt_p)
10332 : {
10333 61197 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10334 61197 : return true;
10335 : }
10336 : /* For SLP reductions we vectorize the epilogue for all involved stmts
10337 : together. For SLP reduction chains we only get here once. */
10338 23528 : if (SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_group
10339 23229 : && slp_index != 0)
10340 : return true;
10341 23048 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
10342 23048 : if (VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
10343 23048 : || VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
10344 : return true;
10345 :
10346 22071 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10347 22071 : || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10348 22062 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
10349 : slp_node_instance,
10350 : LOOP_VINFO_MAIN_EXIT (loop_vinfo));
10351 :
10352 : /* If early break we only have to materialize the reduction on the merge
10353 : block, but we have to find an alternate exit first. */
10354 22071 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
10355 : {
10356 28 : slp_tree phis_node = slp_node_instance->reduc_phis;
10357 28 : stmt_info = SLP_TREE_REPRESENTATIVE (phis_node);
10358 89 : for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
10359 28 : if (exit != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
10360 : {
10361 23 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10362 : phis_node, slp_node_instance,
10363 : exit);
10364 23 : break;
10365 28 : }
10366 28 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10367 9 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10368 : phis_node, slp_node_instance,
10369 : LOOP_VINFO_MAIN_EXIT
10370 : (loop_vinfo));
10371 : }
10372 :
10373 : return true;
10374 : }
10375 :
10376 : /* If STMT is not relevant and it is a simple assignment and its inputs are
10377 : invariant then it can remain in place, unvectorized. The original last
10378 : scalar value that it computes will be used. */
10379 203224 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
10380 : {
10381 0 : gcc_assert (is_simple_and_all_uses_invariant (stmt_info, loop_vinfo));
10382 0 : if (dump_enabled_p ())
10383 0 : dump_printf_loc (MSG_NOTE, vect_location,
10384 : "statement is simple and uses invariant. Leaving in "
10385 : "place.\n");
10386 : return true;
10387 : }
10388 :
10389 203224 : gcc_assert (slp_index >= 0);
10390 :
10391 203224 : int num_vec = vect_get_num_copies (vinfo, slp_node);
10392 203224 : poly_uint64 pos = slp_index;
10393 203224 : if (loop_vinfo)
10394 : {
10395 : /* Get the last occurrence of the scalar index from the concatenation of
10396 : all the slp vectors. Calculate which slp vector it is and the index
10397 : within. */
10398 80182 : int num_scalar = SLP_TREE_LANES (slp_node);
10399 80182 : pos += (num_vec * nunits) - num_scalar;
10400 : }
10401 :
10402 : /* Calculate which vector contains the result, and which lane of
10403 : that vector we need. */
10404 203224 : if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
10405 : {
10406 : if (dump_enabled_p ())
10407 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10408 : "Cannot determine which vector holds the"
10409 : " final result.\n");
10410 : return false;
10411 : }
10412 :
10413 203224 : if (!vec_stmt_p)
10414 : {
10415 : /* No transformation required. */
10416 186370 : if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
10417 : {
10418 31920 : if (SLP_TREE_LANES (slp_node) != 1)
10419 : {
10420 19 : if (dump_enabled_p ())
10421 19 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10422 : "can't operate on partial vectors "
10423 : "because an SLP statement is live after "
10424 : "the loop.\n");
10425 19 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10426 : }
10427 31901 : else if (num_vec > 1)
10428 : {
10429 18647 : if (dump_enabled_p ())
10430 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10431 : "can't operate on partial vectors "
10432 : "because ncopies is greater than 1.\n");
10433 18647 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10434 : }
10435 : else
10436 : {
10437 13254 : if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST_ACTIVE, vectype,
10438 : OPTIMIZE_FOR_SPEED))
10439 0 : vect_record_loop_mask (loop_vinfo,
10440 : &LOOP_VINFO_MASKS (loop_vinfo),
10441 : 1, vectype, NULL);
10442 13254 : else if (can_vec_extract_var_idx_p (
10443 13254 : TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
10444 0 : vect_record_loop_len (loop_vinfo,
10445 : &LOOP_VINFO_LENS (loop_vinfo),
10446 : 1, vectype, 1);
10447 : else
10448 : {
10449 13254 : if (dump_enabled_p ())
10450 752 : dump_printf_loc (
10451 752 : MSG_MISSED_OPTIMIZATION, vect_location,
10452 : "can't operate on partial vectors "
10453 : "because the target doesn't support extract "
10454 : "last reduction.\n");
10455 13254 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10456 : }
10457 : }
10458 : }
10459 31920 : record_stmt_cost (cost_vec, 1, vec_to_scalar, slp_node,
10460 : 0, loop_vinfo ? vect_epilogue : vect_body);
10461 186370 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10462 186370 : return true;
10463 : }
10464 :
10465 : /* Use the lhs of the original scalar statement. */
10466 16854 : gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
10467 16854 : if (dump_enabled_p ())
10468 1048 : dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
10469 : "stmt %G", stmt);
10470 :
10471 16854 : lhs = gimple_get_lhs (stmt);
10472 16854 : lhs_type = TREE_TYPE (lhs);
10473 :
10474 16854 : bitsize = vector_element_bits_tree (vectype);
10475 :
10476 : /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
10477 16854 : gcc_assert (!loop_vinfo
10478 : || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10479 : && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10480 : || SLP_TREE_LANES (slp_node) == 1));
10481 :
10482 : /* Get the correct slp vectorized stmt. */
10483 16854 : tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
10484 :
10485 : /* In case we need to early break vectorize also get the first stmt. */
10486 16854 : tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
10487 :
10488 : /* Get entry to use. */
10489 16854 : tree bitstart = bitsize_int (vec_index);
10490 16854 : bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
10491 :
10492 16854 : if (loop_vinfo)
10493 : {
10494 : /* Ensure the VEC_LHS for lane extraction stmts satisfy loop-closed PHI
10495 : requirement, insert one phi node for it. It looks like:
10496 : loop;
10497 : BB:
10498 : # lhs' = PHI <lhs>
10499 : ==>
10500 : loop;
10501 : BB:
10502 : # vec_lhs' = PHI <vec_lhs>
10503 : new_tree = lane_extract <vec_lhs', ...>;
10504 : lhs' = new_tree; */
10505 :
10506 2882 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10507 : /* Check if we have a loop where the chosen exit is not the main exit,
10508 : in these cases for an early break we restart the iteration the vector code
10509 : did. For the live values we want the value at the start of the iteration
10510 : rather than at the end. */
10511 2882 : edge main_e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
10512 2882 : bool all_exits_as_early_p = LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo);
10513 12106 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10514 9224 : if (!is_gimple_debug (use_stmt)
10515 9224 : && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
10516 2799 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10517 : {
10518 2799 : edge e = gimple_phi_arg_edge (as_a <gphi *> (use_stmt),
10519 2799 : phi_arg_index_from_use (use_p));
10520 2799 : gcc_assert (loop_exit_edge_p (loop, e));
10521 2799 : bool main_exit_edge = e == main_e;
10522 2799 : tree tmp_vec_lhs = vec_lhs;
10523 2799 : tree tmp_bitstart = bitstart;
10524 :
10525 : /* For early exit where the exit is not in the BB that leads
10526 : to the latch then we're restarting the iteration in the
10527 : scalar loop. So get the first live value. */
10528 2799 : bool early_break_first_element_p
10529 2799 : = all_exits_as_early_p || !main_exit_edge;
10530 2799 : if (early_break_first_element_p)
10531 : {
10532 194 : tmp_vec_lhs = vec_lhs0;
10533 194 : tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
10534 : }
10535 :
10536 2799 : gimple_stmt_iterator exit_gsi;
10537 2799 : tree new_tree
10538 2799 : = vectorizable_live_operation_1 (loop_vinfo,
10539 : e->dest, vectype,
10540 : slp_node, bitsize,
10541 : tmp_bitstart, tmp_vec_lhs,
10542 : lhs_type, &exit_gsi);
10543 :
10544 2799 : auto gsi = gsi_for_stmt (use_stmt);
10545 2799 : tree lhs_phi = gimple_phi_result (use_stmt);
10546 2799 : remove_phi_node (&gsi, false);
10547 2799 : gimple *copy = gimple_build_assign (lhs_phi, new_tree);
10548 2799 : gsi_insert_before (&exit_gsi, copy, GSI_SAME_STMT);
10549 2799 : break;
10550 2882 : }
10551 :
10552 : /* There a no further out-of-loop uses of lhs by LC-SSA construction. */
10553 9307 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10554 6425 : gcc_assert (is_gimple_debug (use_stmt)
10555 2882 : || flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)));
10556 : }
10557 : else
10558 : {
10559 : /* For basic-block vectorization simply insert the lane-extraction. */
10560 13972 : tree bftype = TREE_TYPE (vectype);
10561 13972 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10562 18 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10563 13972 : tree new_tree = build3 (BIT_FIELD_REF, bftype,
10564 : vec_lhs, bitsize, bitstart);
10565 13972 : gimple_seq stmts = NULL;
10566 13972 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10567 : &stmts, true, NULL_TREE);
10568 13972 : if (TREE_CODE (new_tree) == SSA_NAME
10569 27944 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
10570 0 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = 1;
10571 13972 : gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
10572 13972 : if (TREE_CODE (vec_lhs) != SSA_NAME
10573 13972 : || SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
10574 13972 : || (vinfo->bbs[0] != gimple_bb (vec_stmt)
10575 9745 : && dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
10576 9745 : gimple_bb (vec_stmt))))
10577 2 : vinfo->insert_seq_on_entry (stmt_info, stmts);
10578 13970 : else if (is_a <gphi *> (vec_stmt))
10579 : {
10580 1384 : gimple_stmt_iterator si = gsi_after_labels (gimple_bb (vec_stmt));
10581 1384 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10582 : }
10583 : else
10584 : {
10585 12586 : gimple_stmt_iterator si = gsi_for_stmt (vec_stmt);
10586 12586 : gsi_insert_seq_after (&si, stmts, GSI_SAME_STMT);
10587 : }
10588 :
10589 : /* Replace use of lhs with newly computed result. If the use stmt is a
10590 : single arg PHI, just replace all uses of PHI result. It's necessary
10591 : because lcssa PHI defining lhs may be before newly inserted stmt. */
10592 13972 : use_operand_p use_p;
10593 13972 : stmt_vec_info use_stmt_info;
10594 76780 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10595 62808 : if (!is_gimple_debug (use_stmt)
10596 62808 : && (!(use_stmt_info = vinfo->lookup_stmt (use_stmt))
10597 46962 : || !PURE_SLP_STMT (use_stmt_info)))
10598 : {
10599 : /* ??? This can happen when the live lane ends up being
10600 : rooted in a vector construction code-generated by an
10601 : external SLP node (and code-generation for that already
10602 : happened).
10603 : Doing this is what would happen if that vector CTOR
10604 : were not code-generated yet so it is not too bad.
10605 : ??? In fact we'd likely want to avoid this situation
10606 : in the first place. */
10607 25801 : if (TREE_CODE (new_tree) == SSA_NAME
10608 25801 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10609 25801 : && gimple_code (use_stmt) != GIMPLE_PHI
10610 48463 : && !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (new_tree),
10611 : use_stmt))
10612 : {
10613 0 : if (dump_enabled_p ())
10614 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10615 : "Using original scalar computation for "
10616 : "live lane because use precedes vector "
10617 : "def\n");
10618 0 : continue;
10619 : }
10620 54070 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10621 : {
10622 : /* ??? It can also happen that we end up pulling a def into
10623 : a loop where replacing out-of-loop uses would require
10624 : a new LC SSA PHI node. Retain the original scalar in
10625 : those cases as well. PR98064. */
10626 27035 : edge e;
10627 27035 : if (TREE_CODE (new_tree) == SSA_NAME
10628 27035 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10629 27035 : && TREE_CODE (vec_lhs) == SSA_NAME
10630 27035 : && !SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
10631 27035 : && (gimple_bb (use_stmt)->loop_father
10632 27035 : != gimple_bb (vec_stmt)->loop_father)
10633 : /* But a replacement in a LC PHI is OK. This happens
10634 : in gcc.dg/vect/bb-slp-57.c for example. */
10635 3054 : && (gimple_code (use_stmt) != GIMPLE_PHI
10636 1207 : || (((e = phi_arg_edge_from_use (use_p)), true)
10637 1207 : && !loop_exit_edge_p
10638 1207 : (gimple_bb (vec_stmt)->loop_father, e)))
10639 29612 : && !flow_loop_nested_p (gimple_bb (vec_stmt)->loop_father,
10640 2577 : gimple_bb (use_stmt)->loop_father))
10641 : {
10642 0 : if (dump_enabled_p ())
10643 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10644 : "Using original scalar computation for "
10645 : "live lane because there is an "
10646 : "out-of-loop definition for it\n");
10647 0 : continue;
10648 : }
10649 27035 : SET_USE (use_p, new_tree);
10650 : }
10651 25801 : update_stmt (use_stmt);
10652 13972 : }
10653 : }
10654 :
10655 : return true;
10656 287949 : }
10657 :
10658 : /* Given loop represented by LOOP_VINFO, return true if computation of
10659 : LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
10660 : otherwise. */
10661 :
10662 : static bool
10663 64347 : loop_niters_no_overflow (loop_vec_info loop_vinfo)
10664 : {
10665 64347 : gcc_assert (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo));
10666 :
10667 : /* Constant case. */
10668 64347 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
10669 : {
10670 37967 : tree cst_niters = LOOP_VINFO_NITERS (loop_vinfo);
10671 37967 : tree cst_nitersm1 = LOOP_VINFO_NITERSM1 (loop_vinfo);
10672 :
10673 37967 : gcc_assert (TREE_CODE (cst_niters) == INTEGER_CST);
10674 37967 : gcc_assert (TREE_CODE (cst_nitersm1) == INTEGER_CST);
10675 37967 : if (wi::to_widest (cst_nitersm1) < wi::to_widest (cst_niters))
10676 : return true;
10677 : }
10678 :
10679 26380 : widest_int max;
10680 26380 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10681 : /* Check the upper bound of loop niters. */
10682 26380 : if (get_max_loop_iterations (loop, &max))
10683 : {
10684 26380 : tree type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
10685 26380 : signop sgn = TYPE_SIGN (type);
10686 26380 : widest_int type_max = widest_int::from (wi::max_value (type), sgn);
10687 26380 : if (max < type_max)
10688 26175 : return true;
10689 26380 : }
10690 : return false;
10691 64347 : }
10692 :
10693 : /* Return a mask type with half the number of elements as OLD_TYPE,
10694 : given that it should have mode NEW_MODE. */
10695 :
10696 : tree
10697 4699 : vect_halve_mask_nunits (tree old_type, machine_mode new_mode)
10698 : {
10699 4699 : poly_uint64 nunits = exact_div (TYPE_VECTOR_SUBPARTS (old_type), 2);
10700 4699 : return build_truth_vector_type_for_mode (nunits, new_mode);
10701 : }
10702 :
10703 : /* Return a mask type with twice as many elements as OLD_TYPE,
10704 : given that it should have mode NEW_MODE. */
10705 :
10706 : tree
10707 7028 : vect_double_mask_nunits (tree old_type, machine_mode new_mode)
10708 : {
10709 7028 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (old_type) * 2;
10710 7028 : return build_truth_vector_type_for_mode (nunits, new_mode);
10711 : }
10712 :
10713 : /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
10714 : contain a sequence of NVECTORS masks that each control a vector of type
10715 : VECTYPE. If SCALAR_MASK is nonnull, the fully-masked loop would AND
10716 : these vector masks with the vector version of SCALAR_MASK. */
10717 :
10718 : void
10719 107752 : vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
10720 : unsigned int nvectors, tree vectype, tree scalar_mask)
10721 : {
10722 107752 : gcc_assert (nvectors != 0);
10723 :
10724 107752 : if (scalar_mask)
10725 : {
10726 5051 : scalar_cond_masked_key cond (scalar_mask, nvectors);
10727 5051 : loop_vinfo->scalar_cond_masked_set.add (cond);
10728 : }
10729 :
10730 107752 : masks->mask_set.add (std::make_pair (vectype, nvectors));
10731 107752 : }
10732 :
10733 : /* Given a complete set of masks MASKS, extract mask number INDEX
10734 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10735 : where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
10736 :
10737 : See the comment above vec_loop_masks for more details about the mask
10738 : arrangement. */
10739 :
10740 : tree
10741 232 : vect_get_loop_mask (loop_vec_info loop_vinfo,
10742 : gimple_stmt_iterator *gsi, vec_loop_masks *masks,
10743 : unsigned int nvectors, tree vectype, unsigned int index)
10744 : {
10745 232 : if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10746 : == vect_partial_vectors_while_ult)
10747 : {
10748 0 : rgroup_controls *rgm = &(masks->rgc_vec)[nvectors - 1];
10749 0 : tree mask_type = rgm->type;
10750 :
10751 : /* Populate the rgroup's mask array, if this is the first time we've
10752 : used it. */
10753 0 : if (rgm->controls.is_empty ())
10754 : {
10755 0 : rgm->controls.safe_grow_cleared (nvectors, true);
10756 0 : for (unsigned int i = 0; i < nvectors; ++i)
10757 : {
10758 0 : tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
10759 : /* Provide a dummy definition until the real one is available. */
10760 0 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10761 0 : rgm->controls[i] = mask;
10762 : }
10763 : }
10764 :
10765 0 : tree mask = rgm->controls[index];
10766 0 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
10767 0 : TYPE_VECTOR_SUBPARTS (vectype)))
10768 : {
10769 : /* A loop mask for data type X can be reused for data type Y
10770 : if X has N times more elements than Y and if Y's elements
10771 : are N times bigger than X's. In this case each sequence
10772 : of N elements in the loop mask will be all-zero or all-one.
10773 : We can then view-convert the mask so that each sequence of
10774 : N elements is replaced by a single element. */
10775 0 : gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type),
10776 : TYPE_VECTOR_SUBPARTS (vectype)));
10777 0 : gimple_seq seq = NULL;
10778 0 : mask_type = truth_type_for (vectype);
10779 0 : mask = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, mask);
10780 0 : if (seq)
10781 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10782 : }
10783 : return mask;
10784 : }
10785 232 : else if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10786 : == vect_partial_vectors_avx512)
10787 : {
10788 : /* The number of scalars per iteration and the number of vectors are
10789 : both compile-time constants. */
10790 232 : unsigned int nscalars_per_iter
10791 232 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10792 232 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10793 :
10794 232 : rgroup_controls *rgm = &masks->rgc_vec[nscalars_per_iter - 1];
10795 :
10796 : /* The stored nV is dependent on the mask type produced. */
10797 232 : gcc_assert (exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10798 : TYPE_VECTOR_SUBPARTS (rgm->type)).to_constant ()
10799 : == rgm->factor);
10800 232 : nvectors = rgm->factor;
10801 :
10802 : /* Populate the rgroup's mask array, if this is the first time we've
10803 : used it. */
10804 232 : if (rgm->controls.is_empty ())
10805 : {
10806 26 : rgm->controls.safe_grow_cleared (nvectors, true);
10807 144 : for (unsigned int i = 0; i < nvectors; ++i)
10808 : {
10809 92 : tree mask = make_temp_ssa_name (rgm->type, NULL, "loop_mask");
10810 : /* Provide a dummy definition until the real one is available. */
10811 92 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10812 92 : rgm->controls[i] = mask;
10813 : }
10814 : }
10815 232 : if (known_eq (TYPE_VECTOR_SUBPARTS (rgm->type),
10816 : TYPE_VECTOR_SUBPARTS (vectype)))
10817 170 : return rgm->controls[index];
10818 :
10819 : /* Split the vector if needed. Since we are dealing with integer mode
10820 : masks with AVX512 we can operate on the integer representation
10821 : performing the whole vector shifting. */
10822 62 : unsigned HOST_WIDE_INT factor;
10823 62 : bool ok = constant_multiple_p (TYPE_VECTOR_SUBPARTS (rgm->type),
10824 62 : TYPE_VECTOR_SUBPARTS (vectype), &factor);
10825 0 : gcc_assert (ok);
10826 62 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (rgm->type)) == MODE_INT);
10827 62 : tree mask_type = truth_type_for (vectype);
10828 62 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (mask_type)) == MODE_INT);
10829 62 : unsigned vi = index / factor;
10830 62 : unsigned vpart = index % factor;
10831 62 : tree vec = rgm->controls[vi];
10832 62 : gimple_seq seq = NULL;
10833 62 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR,
10834 62 : lang_hooks.types.type_for_mode
10835 62 : (TYPE_MODE (rgm->type), 1), vec);
10836 : /* For integer mode masks simply shift the right bits into position. */
10837 62 : if (vpart != 0)
10838 47 : vec = gimple_build (&seq, RSHIFT_EXPR, TREE_TYPE (vec), vec,
10839 : build_int_cst (integer_type_node,
10840 94 : (TYPE_VECTOR_SUBPARTS (vectype)
10841 47 : * vpart)));
10842 62 : vec = gimple_convert (&seq, lang_hooks.types.type_for_mode
10843 62 : (TYPE_MODE (mask_type), 1), vec);
10844 62 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, vec);
10845 62 : if (seq)
10846 62 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10847 : return vec;
10848 : }
10849 : else
10850 0 : gcc_unreachable ();
10851 : }
10852 :
10853 : /* Record that LOOP_VINFO would need LENS to contain a sequence of NVECTORS
10854 : lengths for controlling an operation on VECTYPE. The operation splits
10855 : each element of VECTYPE into FACTOR separate subelements, measuring the
10856 : length as a number of these subelements. */
10857 :
10858 : void
10859 0 : vect_record_loop_len (loop_vec_info loop_vinfo, vec_loop_lens *lens,
10860 : unsigned int nvectors, tree vectype, unsigned int factor)
10861 : {
10862 0 : gcc_assert (nvectors != 0);
10863 0 : if (lens->length () < nvectors)
10864 0 : lens->safe_grow_cleared (nvectors, true);
10865 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10866 :
10867 : /* The number of scalars per iteration, scalar occupied bytes and
10868 : the number of vectors are both compile-time constants. */
10869 0 : unsigned int nscalars_per_iter
10870 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10871 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10872 :
10873 0 : if (rgl->max_nscalars_per_iter < nscalars_per_iter)
10874 : {
10875 : /* For now, we only support cases in which all loads and stores fall back
10876 : to VnQI or none do. */
10877 0 : gcc_assert (!rgl->max_nscalars_per_iter
10878 : || (rgl->factor == 1 && factor == 1)
10879 : || (rgl->max_nscalars_per_iter * rgl->factor
10880 : == nscalars_per_iter * factor));
10881 0 : rgl->max_nscalars_per_iter = nscalars_per_iter;
10882 0 : rgl->type = vectype;
10883 0 : rgl->factor = factor;
10884 : }
10885 0 : }
10886 :
10887 : /* Given a complete set of lengths LENS, extract length number INDEX
10888 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10889 : where 0 <= INDEX < NVECTORS. Return a value that contains FACTOR
10890 : multiplied by the number of elements that should be processed.
10891 : Insert any set-up statements before GSI. */
10892 :
10893 : tree
10894 0 : vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10895 : vec_loop_lens *lens, unsigned int nvectors, tree vectype,
10896 : unsigned int index, unsigned int factor, bool adjusted)
10897 : {
10898 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10899 0 : bool use_bias_adjusted_len =
10900 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) != 0;
10901 :
10902 : /* Populate the rgroup's len array, if this is the first time we've
10903 : used it. */
10904 0 : if (rgl->controls.is_empty ())
10905 : {
10906 0 : rgl->controls.safe_grow_cleared (nvectors, true);
10907 0 : for (unsigned int i = 0; i < nvectors; ++i)
10908 : {
10909 0 : tree len_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
10910 0 : gcc_assert (len_type != NULL_TREE);
10911 :
10912 0 : tree len = make_temp_ssa_name (len_type, NULL, "loop_len");
10913 :
10914 : /* Provide a dummy definition until the real one is available. */
10915 0 : SSA_NAME_DEF_STMT (len) = gimple_build_nop ();
10916 0 : rgl->controls[i] = len;
10917 :
10918 0 : if (use_bias_adjusted_len)
10919 : {
10920 0 : gcc_assert (i == 0);
10921 0 : tree adjusted_len =
10922 0 : make_temp_ssa_name (len_type, NULL, "adjusted_loop_len");
10923 0 : SSA_NAME_DEF_STMT (adjusted_len) = gimple_build_nop ();
10924 0 : rgl->bias_adjusted_ctrl = adjusted_len;
10925 : }
10926 : }
10927 : }
10928 :
10929 0 : if (use_bias_adjusted_len && adjusted)
10930 0 : return rgl->bias_adjusted_ctrl;
10931 :
10932 0 : tree loop_len = rgl->controls[index];
10933 0 : if (rgl->factor == 1 && factor == 1)
10934 : {
10935 0 : poly_int64 nunits1 = TYPE_VECTOR_SUBPARTS (rgl->type);
10936 0 : poly_int64 nunits2 = TYPE_VECTOR_SUBPARTS (vectype);
10937 0 : if (maybe_ne (nunits1, nunits2))
10938 : {
10939 : /* A loop len for data type X can be reused for data type Y
10940 : if X has N times more elements than Y and if Y's elements
10941 : are N times bigger than X's. */
10942 0 : gcc_assert (multiple_p (nunits1, nunits2));
10943 0 : factor = exact_div (nunits1, nunits2).to_constant ();
10944 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10945 0 : gimple_seq seq = NULL;
10946 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10947 0 : build_int_cst (iv_type, factor));
10948 0 : if (seq)
10949 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10950 : }
10951 0 : }
10952 0 : else if (factor && rgl->factor != factor)
10953 : {
10954 : /* The number of scalars per iteration, scalar occupied bytes and
10955 : the number of vectors are both compile-time constants. */
10956 0 : unsigned int nscalars_per_iter
10957 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10958 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10959 0 : unsigned int rglvecsize = rgl->factor * rgl->max_nscalars_per_iter;
10960 0 : unsigned int vecsize = nscalars_per_iter * factor;
10961 0 : if (rglvecsize > vecsize)
10962 : {
10963 0 : unsigned int fac = rglvecsize / vecsize;
10964 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10965 0 : gimple_seq seq = NULL;
10966 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10967 0 : build_int_cst (iv_type, fac));
10968 0 : if (seq)
10969 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10970 : }
10971 0 : else if (rglvecsize < vecsize)
10972 : {
10973 0 : unsigned int fac = vecsize / rglvecsize;
10974 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10975 0 : gimple_seq seq = NULL;
10976 0 : loop_len = gimple_build (&seq, MULT_EXPR, iv_type, loop_len,
10977 0 : build_int_cst (iv_type, fac));
10978 0 : if (seq)
10979 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10980 : }
10981 : }
10982 : return loop_len;
10983 : }
10984 :
10985 : /* Generate the tree for the loop len mask and return it. Given the lens,
10986 : nvectors, vectype, index and factor to gen the len mask as below.
10987 :
10988 : tree len_mask = VCOND_MASK_LEN (compare_mask, ones, zero, len, bias)
10989 : */
10990 : tree
10991 0 : vect_gen_loop_len_mask (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10992 : gimple_stmt_iterator *cond_gsi, vec_loop_lens *lens,
10993 : unsigned int nvectors, tree vectype, tree stmt,
10994 : unsigned int index, unsigned int factor)
10995 : {
10996 0 : tree all_one_mask = build_all_ones_cst (vectype);
10997 0 : tree all_zero_mask = build_zero_cst (vectype);
10998 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens, nvectors, vectype, index,
10999 : factor, true);
11000 0 : tree bias = build_int_cst (intQI_type_node,
11001 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo));
11002 0 : tree len_mask = make_temp_ssa_name (TREE_TYPE (stmt), NULL, "vec_len_mask");
11003 0 : gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5, stmt,
11004 : all_one_mask, all_zero_mask, len,
11005 : bias);
11006 0 : gimple_call_set_lhs (call, len_mask);
11007 0 : gsi_insert_before (cond_gsi, call, GSI_SAME_STMT);
11008 :
11009 0 : return len_mask;
11010 : }
11011 :
11012 : /* Update the exit profile counts scaling them from scalar to vector counts.
11013 :
11014 : The calculations are as follows.
11015 :
11016 : let P be the scalar probability that one iteration takes an early exit edge.
11017 : 1 - P is then the probabiliy that the exit is not taken.
11018 :
11019 : Since every vector iteration handles VF scalar iterations, the early exit
11020 : branch can be taken when any of the lanes are true.
11021 :
11022 : We invert this and first calculate the probablity that you reach the latch,
11023 : i.e. the probability that the vector code does not exit is the probability
11024 : Q that none of the lanes are true.
11025 :
11026 : Q = (1 - P) * (1 - P) * ... * (1 - P)
11027 : = (1 - P) ^ VF
11028 :
11029 : and so the probability that at least one lane breaks is 1 - Q,
11030 : or rather 1 - (i - P) ^ VF.
11031 :
11032 : For example, if the scalar break probability is 10% and VF is 4:
11033 :
11034 : P = 0.1
11035 : 1 - P = 0.9
11036 : (1 - P) ^ VF = 0.9 ^ 4 = 0.6561
11037 : 1 - (1 - p) ^ VF = 1 - 0.6561 = 0.3439
11038 :
11039 : So a scalar early-exit edge that is taken 10% of the time becomes a vector
11040 : early-exit edge that is taken about 34.4% of the time, because each vector
11041 : iteration gives the break condition four chances to trigger. */
11042 :
11043 : static void
11044 64392 : vect_update_early_break_profiles (loop_vec_info loop_vinfo)
11045 : {
11046 64392 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11047 62906 : return;
11048 :
11049 1486 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11050 1486 : unsigned int vf = vect_vf_for_cost (loop_vinfo);
11051 :
11052 1486 : auto_vec<std::pair<edge, profile_probability>, 8> updates;
11053 7556 : for (edge e : get_loop_exit_edges (loop))
11054 : {
11055 4485 : if ((e == LOOP_VINFO_MAIN_EXIT (loop_vinfo)
11056 1432 : && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
11057 3143 : || !e->probability.initialized_p ())
11058 1387 : continue;
11059 :
11060 : /* For any early break exits, the probablity of exiting are based on if
11061 : any lane is true, and so are tied to VF since you have VF chances.
11062 : As such scale the scalar profile by VF to the the probability for the
11063 : vector edges using 1 - (1 - p) ^ VF. */
11064 1711 : profile_probability probability
11065 1711 : = profile_probability::always () - e->probability.invert ().pow (vf);
11066 1711 : updates.safe_push ({e, probability});
11067 1486 : }
11068 :
11069 : /* First update the edge counts. */
11070 6169 : for (auto update : updates)
11071 1711 : set_edge_probability_and_rescale_others (update.first, update.second);
11072 :
11073 : /* And then only update the in loop BBs, i.e. the fall through block for
11074 : the early exits. */
11075 6169 : for (auto update : updates)
11076 8555 : for (edge e : update.first->src->succs)
11077 5133 : if (!loop_exit_edge_p (loop, e) && single_pred_p (e->dest))
11078 1711 : e->dest->count = e->count ();
11079 1486 : }
11080 :
11081 : /* Scale profiling counters by estimation for LOOP which is vectorized
11082 : by factor VF.
11083 : If FLAT is true, the loop we started with had unrealistically flat
11084 : profile. */
11085 :
11086 : static void
11087 64392 : scale_profile_for_vect_loop (loop_vec_info loop_vinfo,
11088 : class loop *loop, edge exit_e, unsigned vf,
11089 : bool flat)
11090 : {
11091 : /* First scale any early exits. */
11092 64392 : vect_update_early_break_profiles (loop_vinfo);
11093 :
11094 : /* For flat profiles do not scale down proportionally by VF and only
11095 : cap by known iteration count bounds. */
11096 64392 : if (flat)
11097 : {
11098 36061 : if (dump_file && (dump_flags & TDF_DETAILS))
11099 5626 : fprintf (dump_file,
11100 : "Vectorized loop profile seems flat; not scaling iteration "
11101 : "count down by the vectorization factor %i\n", vf);
11102 36061 : scale_loop_profile (loop, profile_probability::always (),
11103 : get_likely_max_loop_iterations_int (loop));
11104 36061 : return;
11105 : }
11106 : /* Loop body executes VF fewer times and exit increases VF times. */
11107 28331 : profile_count entry_count = loop_preheader_edge (loop)->count ();
11108 :
11109 : /* If we have unreliable loop profile avoid dropping entry
11110 : count below header count. This can happen since loops
11111 : has unrealistically low trip counts. */
11112 28331 : while (vf > 1
11113 29516 : && loop->header->count > entry_count
11114 60122 : && loop->header->count < entry_count * vf)
11115 : {
11116 2275 : if (dump_file && (dump_flags & TDF_DETAILS))
11117 158 : fprintf (dump_file,
11118 : "Vectorization factor %i seems too large for profile "
11119 : "previously believed to be consistent; reducing.\n", vf);
11120 2275 : vf /= 2;
11121 : }
11122 :
11123 28331 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
11124 : {
11125 28331 : if (entry_count.nonzero_p ())
11126 28331 : set_edge_probability_and_rescale_others
11127 28331 : (exit_e,
11128 28331 : entry_count.probability_in (loop->header->count / vf));
11129 : /* Avoid producing very large exit probability when we do not have
11130 : sensible profile. */
11131 0 : else if (exit_e->probability < profile_probability::always () / (vf * 2))
11132 0 : set_edge_probability_and_rescale_others (exit_e,
11133 : exit_e->probability * vf);
11134 : }
11135 :
11136 28331 : loop->latch->count = single_pred_edge (loop->latch)->count ();
11137 :
11138 28331 : scale_loop_profile (loop, profile_probability::always () / vf,
11139 : get_likely_max_loop_iterations_int (loop));
11140 : }
11141 :
11142 : /* Update EPILOGUE's loop_vec_info. EPILOGUE was constructed as a copy of the
11143 : original loop that has now been vectorized.
11144 :
11145 : The inits of the data_references need to be advanced with the number of
11146 : iterations of the main loop. This has been computed in vect_do_peeling and
11147 : is stored in parameter ADVANCE.
11148 :
11149 : Since the loop_vec_info of this EPILOGUE was constructed for the original
11150 : loop, its stmt_vec_infos all point to the original statements. These need
11151 : to be updated to point to their corresponding copies.
11152 :
11153 : The data_reference's connections also need to be updated. Their
11154 : corresponding dr_vec_info need to be reconnected to the EPILOGUE's
11155 : stmt_vec_infos, their statements need to point to their corresponding
11156 : copy. */
11157 :
11158 : static void
11159 6946 : update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
11160 : {
11161 6946 : loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
11162 6946 : hash_map<tree,tree> mapping;
11163 6946 : gimple *orig_stmt, *new_stmt;
11164 6946 : gimple_stmt_iterator epilogue_gsi;
11165 6946 : gphi_iterator epilogue_phi_gsi;
11166 6946 : stmt_vec_info stmt_vinfo = NULL, related_vinfo;
11167 6946 : basic_block *epilogue_bbs = get_loop_body (epilogue);
11168 6946 : unsigned i;
11169 :
11170 6946 : free (LOOP_VINFO_BBS (epilogue_vinfo));
11171 6946 : LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
11172 6946 : LOOP_VINFO_NBBS (epilogue_vinfo) = epilogue->num_nodes;
11173 :
11174 : /* The EPILOGUE loop is a copy of the original loop so they share the same
11175 : gimple UIDs. In this loop we update the loop_vec_info of the EPILOGUE to
11176 : point to the copied statements. */
11177 20838 : for (unsigned i = 0; i < epilogue->num_nodes; ++i)
11178 : {
11179 13892 : for (epilogue_phi_gsi = gsi_start_phis (epilogue_bbs[i]);
11180 35651 : !gsi_end_p (epilogue_phi_gsi); gsi_next (&epilogue_phi_gsi))
11181 : {
11182 21759 : new_stmt = epilogue_phi_gsi.phi ();
11183 :
11184 21759 : gcc_assert (gimple_uid (new_stmt) > 0);
11185 21759 : stmt_vinfo
11186 21759 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
11187 :
11188 21759 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
11189 : }
11190 :
11191 27784 : for (epilogue_gsi = gsi_start_bb (epilogue_bbs[i]);
11192 138114 : !gsi_end_p (epilogue_gsi); gsi_next (&epilogue_gsi))
11193 : {
11194 124222 : new_stmt = gsi_stmt (epilogue_gsi);
11195 124222 : if (is_gimple_debug (new_stmt))
11196 20049 : continue;
11197 :
11198 104173 : gcc_assert (gimple_uid (new_stmt) > 0);
11199 104173 : stmt_vinfo
11200 104173 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
11201 :
11202 104173 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
11203 :
11204 104173 : related_vinfo = STMT_VINFO_RELATED_STMT (stmt_vinfo);
11205 104173 : if (related_vinfo != NULL && related_vinfo != stmt_vinfo)
11206 : {
11207 2197 : gimple *stmt = STMT_VINFO_STMT (related_vinfo);
11208 : /* Set BB such that the assert in
11209 : 'get_initial_defs_for_reduction' is able to determine that
11210 : the BB of the related stmt is inside this loop. */
11211 2197 : gimple_set_bb (stmt,
11212 : gimple_bb (new_stmt));
11213 2197 : related_vinfo = STMT_VINFO_RELATED_STMT (related_vinfo);
11214 2197 : gcc_assert (related_vinfo == NULL
11215 : || related_vinfo == stmt_vinfo);
11216 : }
11217 : }
11218 : }
11219 :
11220 6946 : struct data_reference *dr;
11221 6946 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (epilogue_vinfo);
11222 31380 : FOR_EACH_VEC_ELT (datarefs, i, dr)
11223 : {
11224 24434 : orig_stmt = DR_STMT (dr);
11225 24434 : gcc_assert (gimple_uid (orig_stmt) > 0);
11226 24434 : stmt_vinfo = epilogue_vinfo->stmt_vec_infos[gimple_uid (orig_stmt) - 1];
11227 24434 : DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo);
11228 : }
11229 :
11230 : /* Advance data_reference's with the number of iterations of the previous
11231 : loop and its prologue. */
11232 6946 : vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
11233 :
11234 : /* Remember the advancement made. */
11235 6946 : LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
11236 6946 : }
11237 :
11238 : /* When vectorizing early break statements instructions that happen before
11239 : the early break in the current BB need to be moved to after the early
11240 : break. This function deals with that and assumes that any validity
11241 : checks has already been performed.
11242 :
11243 : While moving the instructions if it encounters a VUSE or VDEF it then
11244 : corrects the VUSES as it moves the statements along. GDEST is the location
11245 : in which to insert the new statements. */
11246 :
11247 : static void
11248 1486 : move_early_exit_stmts (loop_vec_info loop_vinfo)
11249 : {
11250 1486 : DUMP_VECT_SCOPE ("move_early_exit_stmts");
11251 :
11252 1486 : if (LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).is_empty ())
11253 1236 : return;
11254 :
11255 : /* Move all stmts that need moving. */
11256 250 : basic_block dest_bb = LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo);
11257 250 : gimple_stmt_iterator dest_gsi = gsi_after_labels (dest_bb);
11258 :
11259 250 : tree last_seen_vuse = NULL_TREE;
11260 599 : for (gimple *stmt : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo))
11261 : {
11262 : /* We have to update crossed degenerate virtual PHIs. Simply
11263 : elide them. */
11264 349 : if (gphi *vphi = dyn_cast <gphi *> (stmt))
11265 : {
11266 7 : tree vdef = gimple_phi_result (vphi);
11267 7 : tree vuse = gimple_phi_arg_def (vphi, 0);
11268 7 : imm_use_iterator iter;
11269 7 : use_operand_p use_p;
11270 7 : gimple *use_stmt;
11271 23 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
11272 : {
11273 32 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11274 16 : SET_USE (use_p, vuse);
11275 7 : }
11276 7 : auto gsi = gsi_for_stmt (stmt);
11277 7 : remove_phi_node (&gsi, true);
11278 7 : last_seen_vuse = vuse;
11279 7 : continue;
11280 7 : }
11281 :
11282 : /* Check to see if statement is still required for vect or has been
11283 : elided. */
11284 342 : auto stmt_info = loop_vinfo->lookup_stmt (stmt);
11285 342 : if (!stmt_info)
11286 0 : continue;
11287 :
11288 342 : if (dump_enabled_p ())
11289 177 : dump_printf_loc (MSG_NOTE, vect_location, "moving stmt %G", stmt);
11290 :
11291 342 : gimple_stmt_iterator stmt_gsi = gsi_for_stmt (stmt);
11292 342 : gsi_move_before (&stmt_gsi, &dest_gsi, GSI_NEW_STMT);
11293 684 : last_seen_vuse = gimple_vuse (stmt);
11294 : }
11295 :
11296 : /* Update all the stmts with their new reaching VUSES. */
11297 865 : for (auto p : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
11298 : {
11299 281 : if (dump_enabled_p ())
11300 203 : dump_printf_loc (MSG_NOTE, vect_location,
11301 : "updating vuse to %T for load %G",
11302 : last_seen_vuse, p);
11303 281 : gimple_set_vuse (p, last_seen_vuse);
11304 281 : update_stmt (p);
11305 : }
11306 :
11307 : /* And update the LC PHIs on exits. */
11308 1263 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
11309 513 : if (!dominated_by_p (CDI_DOMINATORS, e->src, dest_bb))
11310 293 : if (gphi *phi = get_virtual_phi (e->dest))
11311 543 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, last_seen_vuse);
11312 : }
11313 :
11314 : /* Generate adjustment code for early break scalar IVs filling in the value
11315 : we created earlier on for LOOP_VINFO_EARLY_BRK_NITERS_VAR. */
11316 :
11317 : static void
11318 1486 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
11319 : {
11320 1486 : DUMP_VECT_SCOPE ("vect_update_ivs_after_vectorizer_for_early_breaks");
11321 :
11322 1486 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
11323 : /* If no peeling was done then we have no IV to update. */
11324 1486 : || !LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo))
11325 598 : return;
11326 :
11327 888 : tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
11328 888 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
11329 888 : tree ty_var = TREE_TYPE (phi_var);
11330 888 : auto loop = LOOP_VINFO_LOOP (loop_vinfo);
11331 888 : tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
11332 :
11333 : /* Remove the existing dummy GIMPLE statement and just keep the def. */
11334 888 : gimple *def = SSA_NAME_DEF_STMT (phi_var);
11335 888 : auto def_gsi = gsi_for_stmt (def);
11336 888 : gsi_remove (&def_gsi, true);
11337 :
11338 888 : auto induction_phi = create_phi_node (induc_var, loop->header);
11339 888 : tree induc_def = PHI_RESULT (induction_phi);
11340 :
11341 : /* Create the iv update inside the loop. */
11342 888 : gimple_seq init_stmts = NULL;
11343 888 : gimple_seq stmts = NULL;
11344 888 : gimple_seq iv_stmts = NULL;
11345 888 : tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
11346 :
11347 888 : tree iter_var;
11348 888 : if (POINTER_TYPE_P (ty_var))
11349 0 : iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
11350 : tree_iv_incr);
11351 : else
11352 : {
11353 888 : tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
11354 888 : iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
11355 : }
11356 :
11357 888 : tree init_var = build_zero_cst (ty_var);
11358 888 : if (niters_skip)
11359 0 : init_var = gimple_build (&init_stmts, MINUS_EXPR, ty_var, init_var,
11360 : gimple_convert (&init_stmts, ty_var, niters_skip));
11361 :
11362 888 : add_phi_arg (induction_phi, iter_var,
11363 : loop_latch_edge (loop), UNKNOWN_LOCATION);
11364 888 : add_phi_arg (induction_phi, init_var,
11365 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
11366 :
11367 : /* Find the first insertion point in the BB. */
11368 888 : auto pe = loop_preheader_edge (loop);
11369 :
11370 : /* If we've done any peeling, calculate the peeling adjustment needed to the
11371 : final IV. */
11372 888 : if (niters_skip)
11373 : {
11374 0 : tree induc_type = TREE_TYPE (induc_def);
11375 0 : tree s_induc_type = signed_type_for (induc_type);
11376 0 : induc_def = gimple_build (&iv_stmts, MAX_EXPR, s_induc_type,
11377 : gimple_convert (&iv_stmts, s_induc_type,
11378 : induc_def),
11379 : build_zero_cst (s_induc_type));
11380 0 : auto stmt = gimple_build_assign (phi_var,
11381 : gimple_convert (&iv_stmts, induc_type,
11382 : induc_def));
11383 0 : gimple_seq_add_stmt_without_update (&iv_stmts, stmt);
11384 0 : basic_block exit_bb = NULL;
11385 : /* Identify the early exit merge block. I wish we had stored this. */
11386 0 : for (auto e : get_loop_exit_edges (loop))
11387 0 : if (e != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
11388 : {
11389 0 : exit_bb = e->dest;
11390 0 : break;
11391 0 : }
11392 :
11393 0 : gcc_assert (exit_bb);
11394 0 : auto exit_gsi = gsi_after_labels (exit_bb);
11395 0 : gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
11396 : }
11397 : /* Write the init_stmts in the loop-preheader block. */
11398 888 : auto psi = gsi_last_nondebug_bb (pe->src);
11399 888 : gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
11400 :
11401 : /* Write the adjustments at the end of the iv increment. */
11402 888 : bool insert_after;
11403 888 : gimple_stmt_iterator incr_gsi;
11404 888 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
11405 :
11406 888 : if (insert_after)
11407 0 : gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT);
11408 : else
11409 888 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT);
11410 : }
11411 :
11412 : /* Function vect_transform_loop.
11413 :
11414 : The analysis phase has determined that the loop is vectorizable.
11415 : Vectorize the loop - created vectorized stmts to replace the scalar
11416 : stmts in the loop, and update the loop exit condition.
11417 : Returns scalar epilogue loop if any. */
11418 :
11419 : class loop *
11420 64392 : vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
11421 : {
11422 64392 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11423 64392 : class loop *epilogue = NULL;
11424 64392 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
11425 64392 : int nbbs = loop->num_nodes;
11426 64392 : int i;
11427 64392 : tree niters_vector = NULL_TREE;
11428 64392 : tree step_vector = NULL_TREE;
11429 64392 : tree niters_vector_mult_vf = NULL_TREE;
11430 64392 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11431 64392 : unsigned int lowest_vf = constant_lower_bound (vf);
11432 64392 : gimple *stmt;
11433 64392 : bool check_profitability = false;
11434 64392 : unsigned int th;
11435 64392 : bool flat = maybe_flat_loop_profile (loop);
11436 64392 : bool uncounted_p = LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo);
11437 :
11438 64392 : DUMP_VECT_SCOPE ("vec_transform_loop");
11439 :
11440 64392 : if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11441 57446 : loop_vinfo->shared->check_datarefs ();
11442 :
11443 : /* Use the more conservative vectorization threshold. If the number
11444 : of iterations is constant assume the cost check has been performed
11445 : by our caller. If the threshold makes all loops profitable that
11446 : run at least the (estimated) vectorization factor number of times
11447 : checking is pointless, too. */
11448 64392 : th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
11449 64392 : if (vect_apply_runtime_profitability_check_p (loop_vinfo))
11450 : {
11451 19065 : if (dump_enabled_p ())
11452 179 : dump_printf_loc (MSG_NOTE, vect_location,
11453 : "Profitability threshold is %d loop iterations.\n",
11454 : th);
11455 : check_profitability = true;
11456 : }
11457 :
11458 : /* Make sure there exists a single-predecessor exit bb. Do this before
11459 : versioning. */
11460 64392 : edge e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
11461 64392 : if (! single_pred_p (e->dest) && !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11462 : {
11463 19517 : split_loop_exit_edge (e, true);
11464 19517 : if (dump_enabled_p ())
11465 2419 : dump_printf (MSG_NOTE, "split exit edge\n");
11466 : }
11467 :
11468 : /* Version the loop first, if required, so the profitability check
11469 : comes first. */
11470 :
11471 64392 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
11472 : {
11473 3906 : class loop *sloop
11474 3906 : = vect_loop_versioning (loop_vinfo, loop_vectorized_call);
11475 3906 : sloop->force_vectorize = false;
11476 3906 : check_profitability = false;
11477 : }
11478 :
11479 : /* Make sure there exists a single-predecessor exit bb also on the
11480 : scalar loop copy. Do this after versioning but before peeling
11481 : so CFG structure is fine for both scalar and if-converted loop
11482 : to make slpeel_duplicate_current_defs_from_edges face matched
11483 : loop closed PHI nodes on the exit. */
11484 64392 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
11485 : {
11486 7307 : e = LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo);
11487 7307 : if (! single_pred_p (e->dest))
11488 : {
11489 7009 : split_loop_exit_edge (e, true);
11490 7009 : if (dump_enabled_p ())
11491 1209 : dump_printf (MSG_NOTE, "split exit edge of scalar loop\n");
11492 : }
11493 : }
11494 :
11495 64392 : tree niters = vect_build_loop_niters (loop_vinfo);
11496 64392 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = niters;
11497 64392 : tree nitersm1 = unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo));
11498 64392 : tree advance;
11499 64392 : drs_init_vec orig_drs_init;
11500 64392 : bool niters_no_overflow = uncounted_p ? false /* Not known. */
11501 64347 : : loop_niters_no_overflow (loop_vinfo);
11502 :
11503 64392 : epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
11504 : &step_vector, &niters_vector_mult_vf, th,
11505 : check_profitability, niters_no_overflow,
11506 : &advance);
11507 :
11508 64392 : LOOP_VINFO_IV_INCREMENT (loop_vinfo)
11509 64392 : = vect_get_loop_iv_increment (loop_vinfo);
11510 :
11511 : /* Assign hierarchical discriminators to the vectorized loop. */
11512 64392 : poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11513 64392 : unsigned int vf_int = constant_lower_bound (vf_val);
11514 64392 : if (vf_int > DISCR_MULTIPLICITY_MAX)
11515 : vf_int = DISCR_MULTIPLICITY_MAX;
11516 :
11517 : /* Assign unique copy_id dynamically instead of using hardcoded constants.
11518 : Epilogue and main vectorized loops get different copy_ids. */
11519 64392 : gimple *loop_last = last_nondebug_stmt (loop->header);
11520 64392 : location_t loop_loc
11521 64392 : = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION;
11522 64118 : if (loop_loc != UNKNOWN_LOCATION)
11523 : {
11524 53515 : unsigned int copyid = allocate_copyid_base (loop_loc, 1);
11525 53515 : assign_discriminators_to_loop (loop, vf_int, copyid);
11526 : }
11527 64392 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
11528 64392 : && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
11529 : {
11530 : /* Ifcvt duplicates loop preheader, loop body and produces an basic
11531 : block after loop exit. We need to scale all that. */
11532 90 : basic_block preheader
11533 90 : = loop_preheader_edge (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))->src;
11534 90 : preheader->count
11535 : = preheader->count.apply_probability
11536 90 : (LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11537 90 : scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo),
11538 : LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11539 90 : LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo)->dest->count = preheader->count;
11540 : }
11541 :
11542 64392 : if (niters_vector == NULL_TREE && !uncounted_p)
11543 : {
11544 29360 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
11545 29360 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11546 59532 : && known_eq (lowest_vf, vf))
11547 : {
11548 29355 : niters_vector
11549 29355 : = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
11550 29355 : LOOP_VINFO_INT_NITERS (loop_vinfo) / lowest_vf);
11551 29355 : step_vector = build_one_cst (TREE_TYPE (niters));
11552 : }
11553 822 : else if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11554 2 : vect_gen_vector_loop_niters (loop_vinfo, niters, &niters_vector,
11555 : &step_vector, niters_no_overflow);
11556 : else
11557 : /* vect_do_peeling subtracted the number of peeled prologue
11558 : iterations from LOOP_VINFO_NITERS. */
11559 820 : vect_gen_vector_loop_niters (loop_vinfo, LOOP_VINFO_NITERS (loop_vinfo),
11560 : &niters_vector, &step_vector,
11561 : niters_no_overflow);
11562 : }
11563 :
11564 : /* 1) Make sure the loop header has exactly two entries
11565 : 2) Make sure we have a preheader basic block. */
11566 :
11567 64392 : gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
11568 :
11569 64392 : split_edge (loop_preheader_edge (loop));
11570 :
11571 64392 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11572 : /* This will deal with any possible peeling. */
11573 2 : vect_prepare_for_masked_peels (loop_vinfo);
11574 :
11575 : /* Handle any code motion that we need to for early-break vectorization after
11576 : we've done peeling but just before we start vectorizing. */
11577 64392 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11578 : {
11579 1486 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vinfo);
11580 1486 : move_early_exit_stmts (loop_vinfo);
11581 : }
11582 :
11583 : /* Remove existing clobber stmts and prefetches. */
11584 196473 : for (i = 0; i < nbbs; i++)
11585 : {
11586 132081 : basic_block bb = bbs[i];
11587 1138191 : for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);)
11588 : {
11589 874029 : stmt = gsi_stmt (si);
11590 874029 : if (gimple_clobber_p (stmt)
11591 874029 : || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH))
11592 : {
11593 102 : unlink_stmt_vdef (stmt);
11594 102 : gsi_remove (&si, true);
11595 102 : release_defs (stmt);
11596 : }
11597 : else
11598 873927 : gsi_next (&si);
11599 : }
11600 : }
11601 :
11602 : /* Schedule the SLP instances. */
11603 64392 : if (!loop_vinfo->slp_instances.is_empty ())
11604 : {
11605 64392 : DUMP_VECT_SCOPE ("scheduling SLP instances");
11606 64392 : vect_schedule_slp (loop_vinfo, LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
11607 : false);
11608 : }
11609 :
11610 : /* Generate the loop invariant statements. */
11611 64392 : if (!gimple_seq_empty_p (LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo)))
11612 : {
11613 80 : if (dump_enabled_p ())
11614 26 : dump_printf_loc (MSG_NOTE, vect_location,
11615 : "------>generating loop invariant statements\n");
11616 80 : gimple_stmt_iterator gsi;
11617 80 : gsi = gsi_after_labels (loop_preheader_edge (loop)->src);
11618 80 : gsi_insert_seq_before (&gsi, LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo),
11619 : GSI_CONTINUE_LINKING);
11620 : }
11621 :
11622 : /* Stub out scalar statements that must not survive vectorization and
11623 : were not picked as relevant in any SLP instance.
11624 : Doing this here helps with grouped statements, or statements that
11625 : are involved in patterns. */
11626 196473 : for (i = 0; i < nbbs; i++)
11627 : {
11628 132081 : basic_block bb = bbs[i];
11629 132081 : stmt_vec_info stmt_info;
11630 264162 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
11631 1733656 : !gsi_end_p (gsi); gsi_next (&gsi))
11632 : {
11633 1601575 : gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
11634 6746 : if (!call || !gimple_call_internal_p (call))
11635 1596198 : continue;
11636 5377 : internal_fn ifn = gimple_call_internal_fn (call);
11637 5377 : if (ifn == IFN_MASK_LOAD)
11638 : {
11639 699 : tree lhs = gimple_get_lhs (call);
11640 699 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11641 : {
11642 0 : tree zero = build_zero_cst (TREE_TYPE (lhs));
11643 0 : gimple *new_stmt = gimple_build_assign (lhs, zero);
11644 0 : gsi_replace (&gsi, new_stmt, true);
11645 : }
11646 : }
11647 4678 : else if (conditional_internal_fn_code (ifn) != ERROR_MARK)
11648 : {
11649 2403 : tree lhs = gimple_get_lhs (call);
11650 2403 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11651 : {
11652 0 : tree else_arg
11653 0 : = gimple_call_arg (call, gimple_call_num_args (call) - 1);
11654 0 : gimple *new_stmt = gimple_build_assign (lhs, else_arg);
11655 0 : gsi_replace (&gsi, new_stmt, true);
11656 : }
11657 : }
11658 2275 : else if (ifn == IFN_MASK_CALL
11659 14 : && (stmt_info = loop_vinfo->lookup_stmt (call))
11660 14 : && !STMT_VINFO_RELEVANT_P (stmt_info)
11661 2289 : && !STMT_VINFO_LIVE_P (stmt_info))
11662 : {
11663 : /* With -fno-tree-{dce,dse} we can end up with dead
11664 : .MASK_CALL without side-effects and thus with a (dead)
11665 : LHS. So we cannot assert there is no LHS. */
11666 14 : loop_vinfo->remove_stmt (stmt_info);
11667 : }
11668 : }
11669 : }
11670 :
11671 64392 : if (!uncounted_p)
11672 : {
11673 : /* The vectorization factor is always > 1, so if we use an IV increment of
11674 : 1. A zero NITERS becomes a nonzero NITERS_VECTOR. */
11675 64347 : if (integer_onep (step_vector))
11676 64323 : niters_no_overflow = true;
11677 :
11678 64347 : vect_set_loop_condition (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
11679 : loop_vinfo, niters_vector, step_vector,
11680 : niters_vector_mult_vf, !niters_no_overflow);
11681 : }
11682 :
11683 64392 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
11684 :
11685 : /* True if the final iteration might not handle a full vector's
11686 : worth of scalar iterations. */
11687 128784 : bool final_iter_may_be_partial
11688 64392 : = LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11689 64392 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo);
11690 :
11691 : /* +1 to convert latch counts to loop iteration counts. */
11692 64392 : int bias_for_lowest = 1;
11693 :
11694 : /* When we are peeling for gaps then we take away one scalar iteration
11695 : from the vector loop. Thus we can adjust the upper bound by one
11696 : scalar iteration. But only when we know the bound applies to the
11697 : IV exit test which might not be true when we have multiple exits. */
11698 64392 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11699 125406 : bias_for_lowest -= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
11700 :
11701 64392 : int bias_for_assumed = bias_for_lowest;
11702 64392 : int alignment_npeels = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
11703 64392 : if (alignment_npeels && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
11704 : {
11705 : /* When the amount of peeling is known at compile time, the first
11706 : iteration will have exactly alignment_npeels active elements.
11707 : In the worst case it will have at least one. */
11708 2 : int min_first_active = (alignment_npeels > 0 ? alignment_npeels : 1);
11709 2 : bias_for_lowest += lowest_vf - min_first_active;
11710 2 : bias_for_assumed += assumed_vf - min_first_active;
11711 : }
11712 : /* In these calculations the "- 1" converts loop iteration counts
11713 : back to latch counts. */
11714 64392 : if (loop->any_upper_bound)
11715 : {
11716 64374 : loop_vec_info main_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
11717 64374 : loop->nb_iterations_upper_bound
11718 64374 : = (final_iter_may_be_partial
11719 65864 : ? wi::udiv_ceil (loop->nb_iterations_upper_bound + bias_for_lowest,
11720 2980 : lowest_vf) - 1
11721 62884 : : wi::udiv_floor (loop->nb_iterations_upper_bound + bias_for_lowest,
11722 125768 : lowest_vf) - 1);
11723 64374 : if (main_vinfo
11724 : /* Both peeling for alignment and peeling for gaps can end up
11725 : with the scalar epilogue running for more than VF-1 iterations. */
11726 6946 : && !main_vinfo->peeling_for_alignment
11727 6899 : && !main_vinfo->peeling_for_gaps)
11728 : {
11729 6719 : unsigned int bound;
11730 6719 : poly_uint64 main_iters
11731 6719 : = upper_bound (LOOP_VINFO_VECT_FACTOR (main_vinfo),
11732 : LOOP_VINFO_COST_MODEL_THRESHOLD (main_vinfo));
11733 6719 : main_iters
11734 6719 : = upper_bound (main_iters,
11735 6719 : LOOP_VINFO_VERSIONING_THRESHOLD (main_vinfo));
11736 6719 : if (can_div_away_from_zero_p (main_iters,
11737 6719 : LOOP_VINFO_VECT_FACTOR (loop_vinfo),
11738 : &bound))
11739 6719 : loop->nb_iterations_upper_bound
11740 6719 : = wi::umin ((bound_wide_int) (bound - 1),
11741 6719 : loop->nb_iterations_upper_bound);
11742 : }
11743 : }
11744 64392 : if (loop->any_likely_upper_bound)
11745 64374 : loop->nb_iterations_likely_upper_bound
11746 64374 : = (final_iter_may_be_partial
11747 65864 : ? wi::udiv_ceil (loop->nb_iterations_likely_upper_bound
11748 1490 : + bias_for_lowest, lowest_vf) - 1
11749 62884 : : wi::udiv_floor (loop->nb_iterations_likely_upper_bound
11750 64374 : + bias_for_lowest, lowest_vf) - 1);
11751 64392 : if (loop->any_estimate)
11752 37566 : loop->nb_iterations_estimate
11753 37566 : = (final_iter_may_be_partial
11754 38269 : ? wi::udiv_ceil (loop->nb_iterations_estimate + bias_for_assumed,
11755 1406 : assumed_vf) - 1
11756 36863 : : wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
11757 74429 : assumed_vf) - 1);
11758 64392 : scale_profile_for_vect_loop (loop_vinfo, loop,
11759 : LOOP_VINFO_MAIN_EXIT (loop_vinfo), assumed_vf,
11760 : flat);
11761 :
11762 64392 : if (dump_enabled_p ())
11763 : {
11764 11496 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11765 : {
11766 9935 : dump_printf_loc (MSG_NOTE, vect_location,
11767 : "LOOP VECTORIZED\n");
11768 9935 : if (loop->inner)
11769 344 : dump_printf_loc (MSG_NOTE, vect_location,
11770 : "OUTER LOOP VECTORIZED\n");
11771 9935 : dump_printf (MSG_NOTE, "\n");
11772 : }
11773 : else
11774 1561 : dump_printf_loc (MSG_NOTE, vect_location,
11775 : "LOOP EPILOGUE VECTORIZED (MODE=%s)\n",
11776 1561 : GET_MODE_NAME (loop_vinfo->vector_mode));
11777 : }
11778 :
11779 : /* Loops vectorized with a variable factor won't benefit from
11780 : unrolling/peeling. */
11781 64392 : if (!vf.is_constant ())
11782 : {
11783 : loop->unroll = 1;
11784 : if (dump_enabled_p ())
11785 : dump_printf_loc (MSG_NOTE, vect_location, "Disabling unrolling due to"
11786 : " variable-length vectorization factor\n");
11787 : }
11788 :
11789 : /* When we have unrolled the loop due to a user requested value we should
11790 : leave it up to the RTL unroll heuristics to determine if it's still worth
11791 : while to unroll more. */
11792 64392 : if (LOOP_VINFO_USER_UNROLL (loop_vinfo))
11793 44 : loop->unroll = 0;
11794 :
11795 : /* Free SLP instances here because otherwise stmt reference counting
11796 : won't work. */
11797 64392 : slp_instance instance;
11798 157263 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
11799 92871 : vect_free_slp_instance (instance);
11800 64392 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
11801 : /* Clear-up safelen field since its value is invalid after vectorization
11802 : since vectorized loop can have loop-carried dependencies. */
11803 64392 : loop->safelen = 0;
11804 :
11805 64392 : if (epilogue)
11806 : {
11807 : /* Accumulate past advancements made. */
11808 6946 : if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
11809 91 : advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
11810 : LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
11811 : advance);
11812 6946 : update_epilogue_loop_vinfo (epilogue, advance);
11813 :
11814 6946 : epilogue->simduid = loop->simduid;
11815 6946 : epilogue->force_vectorize = loop->force_vectorize;
11816 6946 : epilogue->dont_vectorize = false;
11817 : }
11818 :
11819 64392 : return epilogue;
11820 64392 : }
11821 :
11822 : /* The code below is trying to perform simple optimization - revert
11823 : if-conversion for masked stores, i.e. if the mask of a store is zero
11824 : do not perform it and all stored value producers also if possible.
11825 : For example,
11826 : for (i=0; i<n; i++)
11827 : if (c[i])
11828 : {
11829 : p1[i] += 1;
11830 : p2[i] = p3[i] +2;
11831 : }
11832 : this transformation will produce the following semi-hammock:
11833 :
11834 : if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
11835 : {
11836 : vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
11837 : vect__12.22_172 = vect__11.19_170 + vect_cst__171;
11838 : MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
11839 : vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
11840 : vect__19.28_184 = vect__18.25_182 + vect_cst__183;
11841 : MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
11842 : }
11843 : */
11844 :
11845 : void
11846 499 : optimize_mask_stores (class loop *loop)
11847 : {
11848 499 : basic_block *bbs = get_loop_body (loop);
11849 499 : unsigned nbbs = loop->num_nodes;
11850 499 : unsigned i;
11851 499 : basic_block bb;
11852 499 : class loop *bb_loop;
11853 499 : gimple_stmt_iterator gsi;
11854 499 : gimple *stmt;
11855 499 : auto_vec<gimple *> worklist;
11856 499 : auto_purge_vect_location sentinel;
11857 :
11858 499 : vect_location = find_loop_location (loop);
11859 : /* Pick up all masked stores in loop if any. */
11860 1996 : for (i = 0; i < nbbs; i++)
11861 : {
11862 998 : bb = bbs[i];
11863 16957 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
11864 14961 : gsi_next (&gsi))
11865 : {
11866 14961 : stmt = gsi_stmt (gsi);
11867 14961 : if (gimple_call_internal_p (stmt, IFN_MASK_STORE))
11868 673 : worklist.safe_push (stmt);
11869 : }
11870 : }
11871 :
11872 499 : free (bbs);
11873 499 : if (worklist.is_empty ())
11874 68 : return;
11875 :
11876 : /* Loop has masked stores. */
11877 1087 : while (!worklist.is_empty ())
11878 : {
11879 656 : gimple *last, *last_store;
11880 656 : edge e, efalse;
11881 656 : tree mask;
11882 656 : basic_block store_bb, join_bb;
11883 656 : gimple_stmt_iterator gsi_to;
11884 656 : tree vdef, new_vdef;
11885 656 : gphi *phi;
11886 656 : tree vectype;
11887 656 : tree zero;
11888 :
11889 656 : last = worklist.pop ();
11890 656 : mask = gimple_call_arg (last, 2);
11891 656 : bb = gimple_bb (last);
11892 : /* Create then_bb and if-then structure in CFG, then_bb belongs to
11893 : the same loop as if_bb. It could be different to LOOP when two
11894 : level loop-nest is vectorized and mask_store belongs to the inner
11895 : one. */
11896 656 : e = split_block (bb, last);
11897 656 : bb_loop = bb->loop_father;
11898 656 : gcc_assert (loop == bb_loop || flow_loop_nested_p (loop, bb_loop));
11899 656 : join_bb = e->dest;
11900 656 : store_bb = create_empty_bb (bb);
11901 656 : add_bb_to_loop (store_bb, bb_loop);
11902 656 : e->flags = EDGE_TRUE_VALUE;
11903 656 : efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
11904 : /* Put STORE_BB to likely part. */
11905 656 : efalse->probability = profile_probability::likely ();
11906 656 : e->probability = efalse->probability.invert ();
11907 656 : store_bb->count = efalse->count ();
11908 656 : make_single_succ_edge (store_bb, join_bb, EDGE_FALLTHRU);
11909 656 : if (dom_info_available_p (CDI_DOMINATORS))
11910 656 : set_immediate_dominator (CDI_DOMINATORS, store_bb, bb);
11911 656 : if (dump_enabled_p ())
11912 328 : dump_printf_loc (MSG_NOTE, vect_location,
11913 : "Create new block %d to sink mask stores.",
11914 : store_bb->index);
11915 : /* Create vector comparison with boolean result. */
11916 656 : vectype = TREE_TYPE (mask);
11917 656 : zero = build_zero_cst (vectype);
11918 656 : stmt = gimple_build_cond (EQ_EXPR, mask, zero, NULL_TREE, NULL_TREE);
11919 656 : gsi = gsi_last_bb (bb);
11920 656 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
11921 : /* Create new PHI node for vdef of the last masked store:
11922 : .MEM_2 = VDEF <.MEM_1>
11923 : will be converted to
11924 : .MEM.3 = VDEF <.MEM_1>
11925 : and new PHI node will be created in join bb
11926 : .MEM_2 = PHI <.MEM_1, .MEM_3>
11927 : */
11928 656 : vdef = gimple_vdef (last);
11929 656 : new_vdef = make_ssa_name (gimple_vop (cfun), last);
11930 656 : gimple_set_vdef (last, new_vdef);
11931 656 : phi = create_phi_node (vdef, join_bb);
11932 656 : add_phi_arg (phi, new_vdef, EDGE_SUCC (store_bb, 0), UNKNOWN_LOCATION);
11933 :
11934 : /* Put all masked stores with the same mask to STORE_BB if possible. */
11935 690 : while (true)
11936 : {
11937 673 : gimple_stmt_iterator gsi_from;
11938 673 : gimple *stmt1 = NULL;
11939 :
11940 : /* Move masked store to STORE_BB. */
11941 673 : last_store = last;
11942 673 : gsi = gsi_for_stmt (last);
11943 673 : gsi_from = gsi;
11944 : /* Shift GSI to the previous stmt for further traversal. */
11945 673 : gsi_prev (&gsi);
11946 673 : gsi_to = gsi_start_bb (store_bb);
11947 673 : gsi_move_before (&gsi_from, &gsi_to);
11948 : /* Setup GSI_TO to the non-empty block start. */
11949 673 : gsi_to = gsi_start_bb (store_bb);
11950 673 : if (dump_enabled_p ())
11951 344 : dump_printf_loc (MSG_NOTE, vect_location,
11952 : "Move stmt to created bb\n%G", last);
11953 : /* Move all stored value producers if possible. */
11954 5049 : while (!gsi_end_p (gsi))
11955 : {
11956 5048 : tree lhs;
11957 5048 : imm_use_iterator imm_iter;
11958 5048 : use_operand_p use_p;
11959 5048 : bool res;
11960 :
11961 : /* Skip debug statements. */
11962 5048 : if (is_gimple_debug (gsi_stmt (gsi)))
11963 : {
11964 7 : gsi_prev (&gsi);
11965 3156 : continue;
11966 : }
11967 5041 : stmt1 = gsi_stmt (gsi);
11968 : /* Do not consider statements writing to memory or having
11969 : volatile operand. */
11970 9905 : if (gimple_vdef (stmt1)
11971 9905 : || gimple_has_volatile_ops (stmt1))
11972 : break;
11973 4864 : gsi_from = gsi;
11974 4864 : gsi_prev (&gsi);
11975 4864 : lhs = gimple_get_lhs (stmt1);
11976 4864 : if (!lhs)
11977 : break;
11978 :
11979 : /* LHS of vectorized stmt must be SSA_NAME. */
11980 4864 : if (TREE_CODE (lhs) != SSA_NAME)
11981 : break;
11982 :
11983 4864 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11984 : {
11985 : /* Remove dead scalar statement. */
11986 3466 : if (has_zero_uses (lhs))
11987 : {
11988 3149 : gsi_remove (&gsi_from, true);
11989 3149 : release_defs (stmt1);
11990 3149 : continue;
11991 : }
11992 : }
11993 :
11994 : /* Check that LHS does not have uses outside of STORE_BB. */
11995 1715 : res = true;
11996 2959 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
11997 : {
11998 1739 : gimple *use_stmt;
11999 1739 : use_stmt = USE_STMT (use_p);
12000 1739 : if (is_gimple_debug (use_stmt))
12001 0 : continue;
12002 1739 : if (gimple_bb (use_stmt) != store_bb)
12003 : {
12004 : res = false;
12005 : break;
12006 : }
12007 1715 : }
12008 1715 : if (!res)
12009 : break;
12010 :
12011 1220 : if (gimple_vuse (stmt1)
12012 1683 : && gimple_vuse (stmt1) != gimple_vuse (last_store))
12013 : break;
12014 :
12015 : /* Can move STMT1 to STORE_BB. */
12016 1220 : if (dump_enabled_p ())
12017 620 : dump_printf_loc (MSG_NOTE, vect_location,
12018 : "Move stmt to created bb\n%G", stmt1);
12019 1220 : gsi_move_before (&gsi_from, &gsi_to);
12020 : /* Shift GSI_TO for further insertion. */
12021 1220 : gsi_prev (&gsi_to);
12022 : }
12023 : /* Put other masked stores with the same mask to STORE_BB. */
12024 673 : if (worklist.is_empty ()
12025 242 : || gimple_call_arg (worklist.last (), 2) != mask
12026 17 : || worklist.last () != stmt1)
12027 : break;
12028 17 : last = worklist.pop ();
12029 17 : }
12030 1312 : add_phi_arg (phi, gimple_vuse (last_store), e, UNKNOWN_LOCATION);
12031 : }
12032 499 : }
12033 :
12034 : /* Decide whether it is possible to use a zero-based induction variable
12035 : when vectorizing LOOP_VINFO with partial vectors. If it is, return
12036 : the value that the induction variable must be able to hold in order
12037 : to ensure that the rgroups eventually have no active vector elements.
12038 : Return -1 otherwise. */
12039 :
12040 : widest_int
12041 48264 : vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo)
12042 : {
12043 48264 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
12044 48264 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
12045 48264 : unsigned HOST_WIDE_INT max_vf = vect_max_vf (loop_vinfo);
12046 :
12047 : /* Calculate the value that the induction variable must be able
12048 : to hit in order to ensure that we end the loop with an all-false mask.
12049 : This involves adding the maximum number of inactive trailing scalar
12050 : iterations. */
12051 48264 : widest_int iv_limit = -1;
12052 48264 : if (max_loop_iterations (loop, &iv_limit))
12053 : {
12054 48264 : if (niters_skip)
12055 : {
12056 : /* Add the maximum number of skipped iterations to the
12057 : maximum iteration count. */
12058 0 : if (TREE_CODE (niters_skip) == INTEGER_CST)
12059 0 : iv_limit += wi::to_widest (niters_skip);
12060 : else
12061 0 : iv_limit += max_vf - 1;
12062 : }
12063 48264 : else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12064 : /* Make a conservatively-correct assumption. */
12065 326 : iv_limit += max_vf - 1;
12066 :
12067 : /* IV_LIMIT is the maximum number of latch iterations, which is also
12068 : the maximum in-range IV value. Round this value down to the previous
12069 : vector alignment boundary and then add an extra full iteration. */
12070 48264 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
12071 48264 : iv_limit = (iv_limit & -(int) known_alignment (vf)) + max_vf;
12072 : }
12073 48264 : return iv_limit;
12074 : }
12075 :
12076 : /* For the given rgroup_controls RGC, check whether an induction variable
12077 : would ever hit a value that produces a set of all-false masks or zero
12078 : lengths before wrapping around. Return true if it's possible to wrap
12079 : around before hitting the desirable value, otherwise return false. */
12080 :
12081 : bool
12082 0 : vect_rgroup_iv_might_wrap_p (loop_vec_info loop_vinfo, rgroup_controls *rgc)
12083 : {
12084 0 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
12085 :
12086 0 : if (iv_limit == -1)
12087 : return true;
12088 :
12089 0 : tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
12090 0 : unsigned int compare_precision = TYPE_PRECISION (compare_type);
12091 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
12092 :
12093 0 : if (wi::min_precision (iv_limit * nitems, UNSIGNED) > compare_precision)
12094 0 : return true;
12095 :
12096 : return false;
12097 0 : }
|