Line data Source code
1 : /* Loop Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 : Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #define INCLUDE_ALGORITHM
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "cfghooks.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "optabs-tree.h"
35 : #include "memmodel.h"
36 : #include "optabs.h"
37 : #include "diagnostic-core.h"
38 : #include "fold-const.h"
39 : #include "stor-layout.h"
40 : #include "cfganal.h"
41 : #include "gimplify.h"
42 : #include "gimple-iterator.h"
43 : #include "gimplify-me.h"
44 : #include "tree-ssa-loop-ivopts.h"
45 : #include "tree-ssa-loop-manip.h"
46 : #include "tree-ssa-loop-niter.h"
47 : #include "tree-ssa-loop.h"
48 : #include "cfgloop.h"
49 : #include "tree-scalar-evolution.h"
50 : #include "tree-vectorizer.h"
51 : #include "gimple-fold.h"
52 : #include "cgraph.h"
53 : #include "tree-cfg.h"
54 : #include "tree-if-conv.h"
55 : #include "internal-fn.h"
56 : #include "tree-vector-builder.h"
57 : #include "vec-perm-indices.h"
58 : #include "tree-eh.h"
59 : #include "case-cfn-macros.h"
60 : #include "langhooks.h"
61 : #include "opts.h"
62 : #include "hierarchical_discriminator.h"
63 :
64 : /* Loop Vectorization Pass.
65 :
66 : This pass tries to vectorize loops.
67 :
68 : For example, the vectorizer transforms the following simple loop:
69 :
70 : short a[N]; short b[N]; short c[N]; int i;
71 :
72 : for (i=0; i<N; i++){
73 : a[i] = b[i] + c[i];
74 : }
75 :
76 : as if it was manually vectorized by rewriting the source code into:
77 :
78 : typedef int __attribute__((mode(V8HI))) v8hi;
79 : short a[N]; short b[N]; short c[N]; int i;
80 : v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
81 : v8hi va, vb, vc;
82 :
83 : for (i=0; i<N/8; i++){
84 : vb = pb[i];
85 : vc = pc[i];
86 : va = vb + vc;
87 : pa[i] = va;
88 : }
89 :
90 : The main entry to this pass is vectorize_loops(), in which
91 : the vectorizer applies a set of analyses on a given set of loops,
92 : followed by the actual vectorization transformation for the loops that
93 : had successfully passed the analysis phase.
94 : Throughout this pass we make a distinction between two types of
95 : data: scalars (which are represented by SSA_NAMES), and memory references
96 : ("data-refs"). These two types of data require different handling both
97 : during analysis and transformation. The types of data-refs that the
98 : vectorizer currently supports are ARRAY_REFS which base is an array DECL
99 : (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
100 : accesses are required to have a simple (consecutive) access pattern.
101 :
102 : Analysis phase:
103 : ===============
104 : The driver for the analysis phase is vect_analyze_loop().
105 : It applies a set of analyses, some of which rely on the scalar evolution
106 : analyzer (scev) developed by Sebastian Pop.
107 :
108 : During the analysis phase the vectorizer records some information
109 : per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
110 : loop, as well as general information about the loop as a whole, which is
111 : recorded in a "loop_vec_info" struct attached to each loop.
112 :
113 : Transformation phase:
114 : =====================
115 : The loop transformation phase scans all the stmts in the loop, and
116 : creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
117 : the loop that needs to be vectorized. It inserts the vector code sequence
118 : just before the scalar stmt S, and records a pointer to the vector code
119 : in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
120 : attached to S). This pointer will be used for the vectorization of following
121 : stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
122 : otherwise, we rely on dead code elimination for removing it.
123 :
124 : For example, say stmt S1 was vectorized into stmt VS1:
125 :
126 : VS1: vb = px[i];
127 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
128 : S2: a = b;
129 :
130 : To vectorize stmt S2, the vectorizer first finds the stmt that defines
131 : the operand 'b' (S1), and gets the relevant vector def 'vb' from the
132 : vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
133 : resulting sequence would be:
134 :
135 : VS1: vb = px[i];
136 : S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
137 : VS2: va = vb;
138 : S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
139 :
140 : Operands that are not SSA_NAMEs, are data-refs that appear in
141 : load/store operations (like 'x[i]' in S1), and are handled differently.
142 :
143 : Target modeling:
144 : =================
145 : Currently the only target specific information that is used is the
146 : size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
147 : Targets that can support different sizes of vectors, for now will need
148 : to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
149 : flexibility will be added in the future.
150 :
151 : Since we only vectorize operations which vector form can be
152 : expressed using existing tree codes, to verify that an operation is
153 : supported, the vectorizer checks the relevant optab at the relevant
154 : machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
155 : the value found is CODE_FOR_nothing, then there's no target support, and
156 : we can't vectorize the stmt.
157 :
158 : For additional information on this project see:
159 : http://gcc.gnu.org/projects/tree-ssa/vectorization.html
160 : */
161 :
162 : static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *,
163 : unsigned *);
164 : static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
165 : gphi **);
166 :
167 :
168 : /* Function vect_is_simple_iv_evolution.
169 :
170 : FORNOW: A simple evolution of an induction variables in the loop is
171 : considered a polynomial evolution. */
172 :
173 : static bool
174 946089 : vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn,
175 : stmt_vec_info stmt_info)
176 : {
177 946089 : tree init_expr;
178 946089 : tree step_expr;
179 946089 : tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
180 946089 : basic_block bb;
181 :
182 : /* When there is no evolution in this loop, the evolution function
183 : is not "simple". */
184 946089 : if (evolution_part == NULL_TREE)
185 : return false;
186 :
187 : /* When the evolution is a polynomial of degree >= 2
188 : the evolution function is not "simple". */
189 830456 : if (tree_is_chrec (evolution_part))
190 : return false;
191 :
192 830456 : step_expr = evolution_part;
193 830456 : init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
194 :
195 830456 : if (dump_enabled_p ())
196 41216 : dump_printf_loc (MSG_NOTE, vect_location, "step: %T, init: %T\n",
197 : step_expr, init_expr);
198 :
199 830456 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
200 830456 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step_expr;
201 830456 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_add;
202 :
203 830456 : if (TREE_CODE (step_expr) != INTEGER_CST
204 72546 : && (TREE_CODE (step_expr) != SSA_NAME
205 60701 : || ((bb = gimple_bb (SSA_NAME_DEF_STMT (step_expr)))
206 60442 : && flow_bb_inside_loop_p (get_loop (cfun, loop_nb), bb))
207 7782 : || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
208 133 : && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))
209 133 : || !flag_associative_math)))
210 895317 : && (TREE_CODE (step_expr) != REAL_CST
211 416 : || !flag_associative_math))
212 : {
213 64771 : if (dump_enabled_p ())
214 3173 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
215 : "step unknown.\n");
216 : return false;
217 : }
218 :
219 : return true;
220 : }
221 :
222 : /* Function vect_is_nonlinear_iv_evolution
223 :
224 : Only support nonlinear induction for integer type
225 : 1. neg
226 : 2. mul by constant
227 : 3. lshift/rshift by constant.
228 :
229 : For neg induction, return a fake step as integer -1. */
230 : static bool
231 177958 : vect_is_nonlinear_iv_evolution (class loop* loop, stmt_vec_info stmt_info,
232 : gphi* loop_phi_node)
233 : {
234 177958 : tree init_expr, ev_expr, result, op1, op2;
235 177958 : gimple* def;
236 :
237 177958 : if (gimple_phi_num_args (loop_phi_node) != 2)
238 : return false;
239 :
240 177958 : init_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_preheader_edge (loop));
241 177958 : ev_expr = PHI_ARG_DEF_FROM_EDGE (loop_phi_node, loop_latch_edge (loop));
242 :
243 : /* Support nonlinear induction only for integer type. */
244 177958 : if (!INTEGRAL_TYPE_P (TREE_TYPE (init_expr)))
245 : return false;
246 :
247 110621 : result = PHI_RESULT (loop_phi_node);
248 :
249 110621 : if (TREE_CODE (ev_expr) != SSA_NAME
250 108295 : || ((def = SSA_NAME_DEF_STMT (ev_expr)), false)
251 110621 : || !is_gimple_assign (def))
252 : return false;
253 :
254 99743 : enum tree_code t_code = gimple_assign_rhs_code (def);
255 99743 : tree step;
256 99743 : switch (t_code)
257 : {
258 3546 : case NEGATE_EXPR:
259 3546 : if (gimple_assign_rhs1 (def) != result)
260 : return false;
261 3546 : step = build_int_cst (TREE_TYPE (init_expr), -1);
262 3546 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_neg;
263 3546 : break;
264 :
265 12086 : case RSHIFT_EXPR:
266 12086 : case LSHIFT_EXPR:
267 12086 : case MULT_EXPR:
268 12086 : op1 = gimple_assign_rhs1 (def);
269 12086 : op2 = gimple_assign_rhs2 (def);
270 12086 : if (TREE_CODE (op2) != INTEGER_CST
271 8100 : || op1 != result)
272 : return false;
273 7703 : step = op2;
274 7703 : if (t_code == LSHIFT_EXPR)
275 502 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shl;
276 7201 : else if (t_code == RSHIFT_EXPR)
277 6207 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_shr;
278 : /* NEGATE_EXPR and MULT_EXPR are both vect_step_op_mul. */
279 : else
280 994 : STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info) = vect_step_op_mul;
281 : break;
282 :
283 : default:
284 : return false;
285 : }
286 :
287 11249 : STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info) = init_expr;
288 11249 : STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info) = step;
289 :
290 11249 : return true;
291 : }
292 :
293 : /* Returns true if Phi is a first-order recurrence. A first-order
294 : recurrence is a non-reduction recurrence relation in which the value of
295 : the recurrence in the current loop iteration equals a value defined in
296 : the previous iteration. */
297 :
298 : static bool
299 67373 : vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo, class loop *loop,
300 : gphi *phi)
301 : {
302 : /* A nested cycle isn't vectorizable as first order recurrence. */
303 67373 : if (LOOP_VINFO_LOOP (loop_vinfo) != loop)
304 : return false;
305 :
306 : /* Ensure the loop latch definition is from within the loop. */
307 67234 : edge latch = loop_latch_edge (loop);
308 67234 : tree ldef = PHI_ARG_DEF_FROM_EDGE (phi, latch);
309 67234 : if (TREE_CODE (ldef) != SSA_NAME
310 64565 : || SSA_NAME_IS_DEFAULT_DEF (ldef)
311 64499 : || is_a <gphi *> (SSA_NAME_DEF_STMT (ldef))
312 125443 : || !flow_bb_inside_loop_p (loop, gimple_bb (SSA_NAME_DEF_STMT (ldef))))
313 : return false;
314 :
315 57542 : tree def = gimple_phi_result (phi);
316 :
317 : /* Ensure every use_stmt of the phi node is dominated by the latch
318 : definition. */
319 57542 : imm_use_iterator imm_iter;
320 57542 : use_operand_p use_p;
321 73926 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, def)
322 73329 : if (!is_gimple_debug (USE_STMT (use_p))
323 139906 : && (SSA_NAME_DEF_STMT (ldef) == USE_STMT (use_p)
324 45062 : || !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (ldef),
325 : USE_STMT (use_p))))
326 56945 : return false;
327 :
328 : /* First-order recurrence autovectorization needs shuffle vector. */
329 597 : tree scalar_type = TREE_TYPE (def);
330 597 : tree vectype = get_vectype_for_scalar_type (loop_vinfo, scalar_type);
331 597 : if (!vectype)
332 6 : return false;
333 :
334 : return true;
335 : }
336 :
337 : /* Function vect_analyze_scalar_cycles_1.
338 :
339 : Examine the cross iteration def-use cycles of scalar variables
340 : in LOOP. LOOP_VINFO represents the loop that is now being
341 : considered for vectorization (can be LOOP, or an outer-loop
342 : enclosing LOOP). SLP indicates there will be some subsequent
343 : slp analyses or not. */
344 :
345 : static void
346 464232 : vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
347 : {
348 464232 : basic_block bb = loop->header;
349 464232 : auto_vec<stmt_vec_info, 64> worklist;
350 464232 : gphi_iterator gsi;
351 :
352 464232 : DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
353 :
354 : /* First - identify all inductions. Reduction detection assumes that all the
355 : inductions have been identified, therefore, this order must not be
356 : changed. */
357 1658012 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
358 : {
359 1193780 : gphi *phi = gsi.phi ();
360 1193780 : tree access_fn = NULL;
361 1193780 : tree def = PHI_RESULT (phi);
362 1193780 : stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi);
363 :
364 : /* Skip virtual phi's. The data dependences that are associated with
365 : virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
366 2387560 : if (virtual_operand_p (def))
367 416852 : continue;
368 :
369 : /* Skip already analyzed inner loop PHIs of double reductions. */
370 947106 : if (VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
371 1017 : continue;
372 :
373 946089 : if (dump_enabled_p ())
374 43354 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
375 : (gimple *) phi);
376 :
377 946089 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
378 :
379 : /* Analyze the evolution function. */
380 946089 : access_fn = analyze_scalar_evolution (loop, def);
381 946089 : if (dump_enabled_p ())
382 43354 : dump_printf_loc (MSG_NOTE, vect_location,
383 : "Access function of PHI: %T\n", access_fn);
384 946089 : if (access_fn)
385 946089 : STRIP_NOPS (access_fn);
386 :
387 1115250 : if ((!access_fn
388 946089 : || !vect_is_simple_iv_evolution (loop->num, access_fn, stmt_vinfo)
389 765685 : || (LOOP_VINFO_LOOP (loop_vinfo) != loop
390 11448 : && (TREE_CODE (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo))
391 : != INTEGER_CST)))
392 : /* Only handle nonlinear iv for same loop. */
393 1126499 : && (LOOP_VINFO_LOOP (loop_vinfo) != loop
394 177958 : || !vect_is_nonlinear_iv_evolution (loop, stmt_vinfo, phi)))
395 : {
396 169161 : worklist.safe_push (stmt_vinfo);
397 169161 : continue;
398 : }
399 :
400 776928 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo)
401 : != NULL_TREE);
402 776928 : gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo) != NULL_TREE);
403 :
404 776928 : if (dump_enabled_p ())
405 38163 : dump_printf_loc (MSG_NOTE, vect_location, "Detected induction.\n");
406 776928 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
407 :
408 : /* Mark if we have a non-linear IV. */
409 776928 : LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
410 776928 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add;
411 : }
412 :
413 :
414 : /* Second - identify all reductions and nested cycles. */
415 633393 : while (worklist.length () > 0)
416 : {
417 169161 : stmt_vec_info stmt_vinfo = worklist.pop ();
418 169161 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
419 169161 : tree def = PHI_RESULT (phi);
420 :
421 169161 : if (dump_enabled_p ())
422 5191 : dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
423 : (gimple *) phi);
424 :
425 338322 : gcc_assert (!virtual_operand_p (def)
426 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
427 :
428 169161 : gphi *double_reduc;
429 169161 : stmt_vec_info reduc_stmt_info
430 169161 : = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc);
431 169161 : if (reduc_stmt_info && double_reduc)
432 : {
433 1119 : stmt_vec_info inner_phi_info
434 1119 : = loop_vinfo->lookup_stmt (double_reduc);
435 : /* ??? Pass down flag we're the inner loop of a double reduc. */
436 1119 : stmt_vec_info inner_reduc_info
437 1119 : = vect_is_simple_reduction (loop_vinfo, inner_phi_info, NULL);
438 1119 : if (inner_reduc_info)
439 : {
440 1017 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
441 1017 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
442 1017 : STMT_VINFO_REDUC_DEF (inner_phi_info) = inner_reduc_info;
443 1017 : STMT_VINFO_REDUC_DEF (inner_reduc_info) = inner_phi_info;
444 1017 : if (dump_enabled_p ())
445 130 : dump_printf_loc (MSG_NOTE, vect_location,
446 : "Detected double reduction.\n");
447 :
448 1017 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
449 1017 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_double_reduction_def;
450 1017 : STMT_VINFO_DEF_TYPE (inner_phi_info) = vect_nested_cycle;
451 : /* Make it accessible for SLP vectorization. */
452 1017 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
453 : }
454 102 : else if (dump_enabled_p ())
455 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
456 : "Unknown def-use cycle pattern.\n");
457 : }
458 168042 : else if (reduc_stmt_info)
459 : {
460 100669 : if (loop != LOOP_VINFO_LOOP (loop_vinfo))
461 : {
462 2313 : if (dump_enabled_p ())
463 434 : dump_printf_loc (MSG_NOTE, vect_location,
464 : "Detected vectorizable nested cycle.\n");
465 :
466 2313 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
467 : }
468 : else
469 : {
470 98356 : STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
471 98356 : STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
472 98356 : if (dump_enabled_p ())
473 4049 : dump_printf_loc (MSG_NOTE, vect_location,
474 : "Detected reduction.\n");
475 :
476 98356 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
477 98356 : STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def;
478 98356 : LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info);
479 : }
480 : }
481 67373 : else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
482 591 : STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
483 : else
484 66782 : if (dump_enabled_p ())
485 487 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
486 : "Unknown def-use cycle pattern.\n");
487 : }
488 464232 : }
489 :
490 :
491 : /* Function vect_analyze_scalar_cycles.
492 :
493 : Examine the cross iteration def-use cycles of scalar variables, by
494 : analyzing the loop-header PHIs of scalar variables. Classify each
495 : cycle as one of the following: invariant, induction, reduction, unknown.
496 : We do that for the loop represented by LOOP_VINFO, and also to its
497 : inner-loop, if exists.
498 : Examples for scalar cycles:
499 :
500 : Example1: reduction:
501 :
502 : loop1:
503 : for (i=0; i<N; i++)
504 : sum += a[i];
505 :
506 : Example2: induction:
507 :
508 : loop2:
509 : for (i=0; i<N; i++)
510 : a[i] = i; */
511 :
512 : static void
513 458412 : vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
514 : {
515 458412 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
516 :
517 458412 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
518 :
519 : /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
520 : Reductions in such inner-loop therefore have different properties than
521 : the reductions in the nest that gets vectorized:
522 : 1. When vectorized, they are executed in the same order as in the original
523 : scalar loop, so we can't change the order of computation when
524 : vectorizing them.
525 : 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
526 : current checks are too strict. */
527 :
528 458412 : if (loop->inner)
529 5820 : vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
530 458412 : }
531 :
532 : /* Function vect_get_loop_niters.
533 :
534 : Determine how many iterations the loop is executed and place it
535 : in NUMBER_OF_ITERATIONS. Place the number of latch iterations
536 : in NUMBER_OF_ITERATIONSM1. Place the condition under which the
537 : niter information holds in ASSUMPTIONS.
538 :
539 : Return the loop exit conditions. */
540 :
541 :
542 : static vec<gcond *>
543 288733 : vect_get_loop_niters (class loop *loop, const_edge main_exit, tree *assumptions,
544 : tree *number_of_iterations, tree *number_of_iterationsm1)
545 : {
546 288733 : auto_vec<edge> exits = get_loop_exit_edges (loop);
547 288733 : vec<gcond *> conds;
548 577466 : conds.create (exits.length ());
549 288733 : class tree_niter_desc niter_desc;
550 288733 : tree niter_assumptions, niter, may_be_zero;
551 :
552 288733 : *assumptions = boolean_true_node;
553 288733 : *number_of_iterationsm1 = chrec_dont_know;
554 288733 : *number_of_iterations = chrec_dont_know;
555 :
556 288733 : DUMP_VECT_SCOPE ("get_loop_niters");
557 :
558 288733 : if (exits.is_empty ())
559 0 : return conds;
560 :
561 288733 : if (dump_enabled_p ())
562 15000 : dump_printf_loc (MSG_NOTE, vect_location, "Loop has %d exits.\n",
563 : exits.length ());
564 :
565 288733 : edge exit;
566 288733 : unsigned int i;
567 705339 : FOR_EACH_VEC_ELT (exits, i, exit)
568 : {
569 416606 : gcond *cond = get_loop_exit_condition (exit);
570 416606 : if (cond)
571 416572 : conds.safe_push (cond);
572 :
573 416606 : if (dump_enabled_p ())
574 16199 : dump_printf_loc (MSG_NOTE, vect_location, "Analyzing exit %d...\n", i);
575 :
576 416606 : if (exit != main_exit)
577 189223 : continue;
578 :
579 288733 : may_be_zero = NULL_TREE;
580 288733 : if (!number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
581 288733 : || chrec_contains_undetermined (niter_desc.niter))
582 61350 : continue;
583 :
584 227383 : niter_assumptions = niter_desc.assumptions;
585 227383 : may_be_zero = niter_desc.may_be_zero;
586 227383 : niter = niter_desc.niter;
587 :
588 227383 : if (may_be_zero && integer_zerop (may_be_zero))
589 : may_be_zero = NULL_TREE;
590 :
591 9523 : if (may_be_zero)
592 : {
593 9523 : if (COMPARISON_CLASS_P (may_be_zero))
594 : {
595 : /* Try to combine may_be_zero with assumptions, this can simplify
596 : computation of niter expression. */
597 9523 : if (niter_assumptions && !integer_nonzerop (niter_assumptions))
598 1016 : niter_assumptions = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
599 : niter_assumptions,
600 : fold_build1 (TRUTH_NOT_EXPR,
601 : boolean_type_node,
602 : may_be_zero));
603 : else
604 8507 : niter = fold_build3 (COND_EXPR, TREE_TYPE (niter), may_be_zero,
605 : build_int_cst (TREE_TYPE (niter), 0),
606 : rewrite_to_non_trapping_overflow (niter));
607 :
608 227383 : may_be_zero = NULL_TREE;
609 : }
610 0 : else if (integer_nonzerop (may_be_zero))
611 : {
612 0 : *number_of_iterationsm1 = build_int_cst (TREE_TYPE (niter), 0);
613 0 : *number_of_iterations = build_int_cst (TREE_TYPE (niter), 1);
614 0 : continue;
615 : }
616 : else
617 0 : continue;
618 : }
619 :
620 : /* Loop assumptions are based off the normal exit. */
621 227383 : *assumptions = niter_assumptions;
622 227383 : *number_of_iterationsm1 = niter;
623 :
624 : /* We want the number of loop header executions which is the number
625 : of latch executions plus one.
626 : ??? For UINT_MAX latch executions this number overflows to zero
627 : for loops like do { n++; } while (n != 0); */
628 227383 : if (niter && !chrec_contains_undetermined (niter))
629 : {
630 227383 : niter = fold_build2 (PLUS_EXPR, TREE_TYPE (niter),
631 : unshare_expr (niter),
632 : build_int_cst (TREE_TYPE (niter), 1));
633 227383 : if (TREE_CODE (niter) == INTEGER_CST
634 125426 : && TREE_CODE (*number_of_iterationsm1) != INTEGER_CST)
635 : {
636 : /* If we manage to fold niter + 1 into INTEGER_CST even when
637 : niter is some complex expression, ensure back
638 : *number_of_iterationsm1 is an INTEGER_CST as well. See
639 : PR113210. */
640 0 : *number_of_iterationsm1
641 0 : = fold_build2 (PLUS_EXPR, TREE_TYPE (niter), niter,
642 : build_minus_one_cst (TREE_TYPE (niter)));
643 : }
644 : }
645 227383 : *number_of_iterations = niter;
646 : }
647 :
648 288733 : if (dump_enabled_p ())
649 15000 : dump_printf_loc (MSG_NOTE, vect_location, "All loop exits successfully analyzed.\n");
650 :
651 288733 : return conds;
652 288733 : }
653 :
654 : /* Determine the main loop exit for the vectorizer. */
655 :
656 : edge
657 502693 : vec_init_loop_exit_info (class loop *loop)
658 : {
659 : /* Before we begin we must first determine which exit is the main one and
660 : which are auxiliary exits. */
661 502693 : auto_vec<edge> exits = get_loop_exit_edges (loop);
662 1000292 : if (exits.length () == 0)
663 : return NULL;
664 497599 : if (exits.length () == 1)
665 323607 : return exits[0];
666 :
667 : /* If we have multiple exits, look for counting IV exit.
668 : Analyze all exits and return the last one we can analyze. */
669 173992 : class tree_niter_desc niter_desc;
670 173992 : edge candidate = NULL;
671 646960 : for (edge exit : exits)
672 : {
673 494949 : if (!get_loop_exit_condition (exit))
674 : {
675 21981 : if (dump_enabled_p ())
676 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
677 : "Unhandled loop exit detected.\n");
678 : return NULL;
679 : }
680 :
681 472968 : if (number_of_iterations_exit_assumptions (loop, exit, &niter_desc, NULL)
682 472968 : && !chrec_contains_undetermined (niter_desc.niter))
683 : {
684 138220 : tree may_be_zero = niter_desc.may_be_zero;
685 138220 : if ((integer_zerop (may_be_zero)
686 : /* As we are handling may_be_zero that's not false by
687 : rewriting niter to may_be_zero ? 0 : niter we require
688 : an empty latch. */
689 483564 : || (single_pred_p (loop->latch)
690 10395 : && exit->src == single_pred (loop->latch)
691 2785 : && (integer_nonzerop (may_be_zero)
692 2785 : || COMPARISON_CLASS_P (may_be_zero))))
693 141005 : && (!candidate
694 5825 : || dominated_by_p (CDI_DOMINATORS, exit->src,
695 5825 : candidate->src)))
696 : candidate = exit;
697 : }
698 : }
699 :
700 : /* If no exit is analyzable by scalar evolution, we return the last exit
701 : under the assummption we are dealing with an uncounted loop. */
702 210830 : if (!candidate && single_pred_p (loop->latch))
703 36838 : candidate = loop_exits_from_bb_p (loop, single_pred (loop->latch));
704 :
705 : return candidate;
706 173992 : }
707 :
708 : /* Function bb_in_loop_p
709 :
710 : Used as predicate for dfs order traversal of the loop bbs. */
711 :
712 : static bool
713 1755231 : bb_in_loop_p (const_basic_block bb, const void *data)
714 : {
715 1755231 : const class loop *const loop = (const class loop *)data;
716 1755231 : if (flow_bb_inside_loop_p (loop, bb))
717 : return true;
718 : return false;
719 : }
720 :
721 :
722 : /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
723 : stmt_vec_info structs for all the stmts in LOOP_IN. */
724 :
725 598181 : _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
726 : : vec_info (vec_info::loop, shared),
727 598181 : loop (loop_in),
728 598181 : num_itersm1 (NULL_TREE),
729 598181 : num_iters (NULL_TREE),
730 598181 : num_iters_unchanged (NULL_TREE),
731 598181 : num_iters_assumptions (NULL_TREE),
732 598181 : vector_costs (nullptr),
733 598181 : scalar_costs (nullptr),
734 598181 : th (0),
735 598181 : versioning_threshold (0),
736 598181 : vectorization_factor (0),
737 598181 : main_loop_edge (nullptr),
738 598181 : skip_main_loop_edge (nullptr),
739 598181 : skip_this_loop_edge (nullptr),
740 598181 : reusable_accumulators (),
741 598181 : suggested_unroll_factor (1),
742 598181 : max_vectorization_factor (0),
743 598181 : mask_skip_niters (NULL_TREE),
744 598181 : mask_skip_niters_pfa_offset (NULL_TREE),
745 598181 : rgroup_compare_type (NULL_TREE),
746 598181 : simd_if_cond (NULL_TREE),
747 598181 : partial_vector_style (vect_partial_vectors_none),
748 598181 : unaligned_dr (NULL),
749 598181 : peeling_for_alignment (0),
750 598181 : ptr_mask (0),
751 598181 : max_spec_read_amount (0),
752 598181 : nonlinear_iv (false),
753 598181 : ivexpr_map (NULL),
754 598181 : scan_map (NULL),
755 598181 : inner_loop_cost_factor (param_vect_inner_loop_cost_factor),
756 598181 : vectorizable (false),
757 598181 : can_use_partial_vectors_p (true),
758 598181 : must_use_partial_vectors_p (false),
759 598181 : using_partial_vectors_p (false),
760 598181 : using_decrementing_iv_p (false),
761 598181 : using_select_vl_p (false),
762 598181 : allow_mutual_alignment (false),
763 598181 : partial_load_store_bias (0),
764 598181 : peeling_for_gaps (false),
765 598181 : peeling_for_niter (false),
766 598181 : early_breaks (false),
767 598181 : loop_iv_cond (NULL),
768 598181 : user_unroll (false),
769 598181 : no_data_dependencies (false),
770 598181 : has_mask_store (false),
771 598181 : scalar_loop_scaling (profile_probability::uninitialized ()),
772 598181 : scalar_loop (NULL),
773 598181 : main_loop_info (NULL),
774 598181 : orig_loop_info (NULL),
775 598181 : epilogue_vinfo (NULL),
776 598181 : drs_advanced_by (NULL_TREE),
777 598181 : vec_loop_main_exit (NULL),
778 598181 : vec_epilogue_loop_main_exit (NULL),
779 598181 : scalar_loop_main_exit (NULL),
780 598181 : early_break_needs_epilogue (false),
781 598181 : early_break_niters_var (NULL)
782 : {
783 : /* CHECKME: We want to visit all BBs before their successors (except for
784 : latch blocks, for which this assertion wouldn't hold). In the simple
785 : case of the loop forms we allow, a dfs order of the BBs would the same
786 : as reversed postorder traversal, so we are safe. */
787 :
788 598181 : bbs = XCNEWVEC (basic_block, loop->num_nodes);
789 1196362 : nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p, bbs,
790 598181 : loop->num_nodes, loop);
791 598181 : gcc_assert (nbbs == loop->num_nodes);
792 :
793 2087650 : for (unsigned int i = 0; i < nbbs; i++)
794 : {
795 1489469 : basic_block bb = bbs[i];
796 1489469 : gimple_stmt_iterator si;
797 :
798 3068498 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
799 : {
800 1579029 : gimple *phi = gsi_stmt (si);
801 1579029 : gimple_set_uid (phi, 0);
802 1579029 : add_stmt (phi);
803 : }
804 :
805 13963147 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
806 : {
807 10984209 : gimple *stmt = gsi_stmt (si);
808 10984209 : gimple_set_uid (stmt, 0);
809 10984209 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
810 4828390 : continue;
811 6155819 : add_stmt (stmt);
812 : /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
813 : third argument is the #pragma omp simd if (x) condition, when 0,
814 : loop shouldn't be vectorized, when non-zero constant, it should
815 : be vectorized normally, otherwise versioned with vectorized loop
816 : done if the condition is non-zero at runtime. */
817 6155819 : if (loop_in->simduid
818 43333 : && is_gimple_call (stmt)
819 4268 : && gimple_call_internal_p (stmt)
820 4141 : && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
821 4137 : && gimple_call_num_args (stmt) >= 3
822 104 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
823 6155923 : && (loop_in->simduid
824 104 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0))))
825 : {
826 104 : tree arg = gimple_call_arg (stmt, 2);
827 104 : if (integer_zerop (arg) || TREE_CODE (arg) == SSA_NAME)
828 104 : simd_if_cond = arg;
829 : else
830 0 : gcc_assert (integer_nonzerop (arg));
831 : }
832 : }
833 : }
834 598181 : }
835 :
836 : /* Free all levels of rgroup CONTROLS. */
837 :
838 : void
839 1495428 : release_vec_loop_controls (vec<rgroup_controls> *controls)
840 : {
841 1495428 : rgroup_controls *rgc;
842 1495428 : unsigned int i;
843 1520116 : FOR_EACH_VEC_ELT (*controls, i, rgc)
844 24688 : rgc->controls.release ();
845 1495428 : controls->release ();
846 1495428 : }
847 :
848 : /* Free all memory used by the _loop_vec_info, as well as all the
849 : stmt_vec_info structs of all the stmts in the loop. */
850 :
851 598181 : _loop_vec_info::~_loop_vec_info ()
852 : {
853 598181 : free (bbs);
854 :
855 598181 : release_vec_loop_controls (&masks.rgc_vec);
856 598181 : release_vec_loop_controls (&lens);
857 602183 : delete ivexpr_map;
858 598503 : delete scan_map;
859 598181 : delete scalar_costs;
860 598181 : delete vector_costs;
861 815814 : for (auto reduc_info : reduc_infos)
862 208858 : delete reduc_info;
863 :
864 : /* When we release an epiloge vinfo that we do not intend to use
865 : avoid clearing AUX of the main loop which should continue to
866 : point to the main loop vinfo since otherwise we'll leak that. */
867 598181 : if (loop->aux == this)
868 63515 : loop->aux = NULL;
869 1196362 : }
870 :
871 : /* Return an invariant or register for EXPR and emit necessary
872 : computations in the LOOP_VINFO loop preheader. */
873 :
874 : tree
875 20853 : cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo, tree expr)
876 : {
877 20853 : if (is_gimple_reg (expr)
878 20853 : || is_gimple_min_invariant (expr))
879 : return expr;
880 :
881 13650 : if (! loop_vinfo->ivexpr_map)
882 4002 : loop_vinfo->ivexpr_map = new hash_map<tree_operand_hash, tree>;
883 13650 : tree &cached = loop_vinfo->ivexpr_map->get_or_insert (expr);
884 13650 : if (! cached)
885 : {
886 8799 : gimple_seq stmts = NULL;
887 8799 : cached = force_gimple_operand (unshare_expr (expr),
888 : &stmts, true, NULL_TREE);
889 8799 : if (stmts)
890 : {
891 8645 : edge e = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
892 8645 : gsi_insert_seq_on_edge_immediate (e, stmts);
893 : }
894 : }
895 13650 : return cached;
896 : }
897 :
898 : /* Return true if we can use CMP_TYPE as the comparison type to produce
899 : all masks required to mask LOOP_VINFO. */
900 :
901 : static bool
902 110571 : can_produce_all_loop_masks_p (loop_vec_info loop_vinfo, tree cmp_type)
903 : {
904 110571 : rgroup_controls *rgm;
905 110571 : unsigned int i;
906 126286 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
907 126286 : if (rgm->type != NULL_TREE
908 126286 : && !direct_internal_fn_supported_p (IFN_WHILE_ULT,
909 : cmp_type, rgm->type,
910 : OPTIMIZE_FOR_SPEED))
911 : return false;
912 : return true;
913 : }
914 :
915 : /* Calculate the maximum number of scalars per iteration for every
916 : rgroup in LOOP_VINFO. */
917 :
918 : static unsigned int
919 23571 : vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo)
920 : {
921 23571 : unsigned int res = 1;
922 23571 : unsigned int i;
923 23571 : rgroup_controls *rgm;
924 56451 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgm)
925 32880 : res = MAX (res, rgm->max_nscalars_per_iter);
926 23571 : return res;
927 : }
928 :
929 : /* Calculate the minimum precision necessary to represent:
930 :
931 : MAX_NITERS * FACTOR
932 :
933 : as an unsigned integer, where MAX_NITERS is the maximum number of
934 : loop header iterations for the original scalar form of LOOP_VINFO. */
935 :
936 : unsigned
937 25992 : vect_min_prec_for_max_niters (loop_vec_info loop_vinfo, unsigned int factor)
938 : {
939 25992 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
940 :
941 : /* Get the maximum number of iterations that is representable
942 : in the counter type. */
943 25992 : tree ni_type;
944 25992 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
945 25992 : ni_type = TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo));
946 : else
947 0 : ni_type = sizetype;
948 25992 : widest_int max_ni = wi::to_widest (TYPE_MAX_VALUE (ni_type)) + 1;
949 :
950 : /* Get a more refined estimate for the number of iterations. */
951 25992 : widest_int max_back_edges;
952 25992 : if (max_loop_iterations (loop, &max_back_edges))
953 25992 : max_ni = wi::smin (max_ni, max_back_edges + 1);
954 :
955 : /* Work out how many bits we need to represent the limit. */
956 25992 : return wi::min_precision (max_ni * factor, UNSIGNED);
957 25992 : }
958 :
959 : /* True if the loop needs peeling or partial vectors when vectorized. */
960 :
961 : static bool
962 157772 : vect_need_peeling_or_partial_vectors_p (loop_vec_info loop_vinfo)
963 : {
964 157772 : unsigned HOST_WIDE_INT const_vf;
965 :
966 157772 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
967 : return true;
968 :
969 13711 : loop_vec_info main_loop_vinfo
970 156457 : = (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
971 156457 : ? LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) : loop_vinfo);
972 156457 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
973 80497 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo) >= 0)
974 : {
975 : /* Work out the (constant) number of iterations that need to be
976 : peeled for reasons other than niters. */
977 80447 : unsigned int peel_niter
978 : = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
979 80447 : return !multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter,
980 80447 : LOOP_VINFO_VECT_FACTOR (loop_vinfo));
981 : }
982 :
983 76010 : if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo)
984 76010 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&const_vf))
985 : {
986 : /* When the number of iterations is a multiple of the vectorization
987 : factor and we are not doing prologue or forced epilogue peeling
988 : the epilogue isn't necessary. */
989 75596 : if (tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
990 151192 : >= (unsigned) exact_log2 (const_vf))
991 1785 : return false;
992 : }
993 :
994 : return true;
995 : }
996 :
997 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
998 : whether we can actually generate the masks required. Return true if so,
999 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_COMPARE_TYPE. */
1000 :
1001 : static bool
1002 23571 : vect_verify_full_masking (loop_vec_info loop_vinfo)
1003 : {
1004 23571 : unsigned int min_ni_width;
1005 :
1006 : /* Use a normal loop if there are no statements that need masking.
1007 : This only happens in rare degenerate cases: it means that the loop
1008 : has no loads, no stores, and no live-out values. */
1009 23571 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1010 : return false;
1011 :
1012 : /* Produce the rgroup controls. */
1013 58240 : for (auto mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1014 : {
1015 34669 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1016 34669 : tree vectype = mask.first;
1017 34669 : unsigned nvectors = mask.second;
1018 :
1019 45767 : if (masks->rgc_vec.length () < nvectors)
1020 25736 : masks->rgc_vec.safe_grow_cleared (nvectors, true);
1021 34669 : rgroup_controls *rgm = &(*masks).rgc_vec[nvectors - 1];
1022 : /* The number of scalars per iteration and the number of vectors are
1023 : both compile-time constants. */
1024 34669 : unsigned int nscalars_per_iter
1025 34669 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1026 34669 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1027 :
1028 34669 : if (rgm->max_nscalars_per_iter < nscalars_per_iter)
1029 : {
1030 27611 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1031 27611 : rgm->type = truth_type_for (vectype);
1032 27611 : rgm->factor = 1;
1033 : }
1034 : }
1035 :
1036 23571 : unsigned int max_nscalars_per_iter
1037 23571 : = vect_get_max_nscalars_per_iter (loop_vinfo);
1038 :
1039 : /* Work out how many bits we need to represent the limit. */
1040 23571 : min_ni_width
1041 23571 : = vect_min_prec_for_max_niters (loop_vinfo, max_nscalars_per_iter);
1042 :
1043 : /* Find a scalar mode for which WHILE_ULT is supported. */
1044 23571 : opt_scalar_int_mode cmp_mode_iter;
1045 23571 : tree cmp_type = NULL_TREE;
1046 23571 : tree iv_type = NULL_TREE;
1047 23571 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1048 23571 : unsigned int iv_precision = UINT_MAX;
1049 :
1050 23571 : if (iv_limit != -1)
1051 23571 : iv_precision = wi::min_precision (iv_limit * max_nscalars_per_iter,
1052 : UNSIGNED);
1053 :
1054 188568 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1055 : {
1056 164997 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1057 164997 : if (cmp_bits >= min_ni_width
1058 164997 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1059 : {
1060 110571 : tree this_type = build_nonstandard_integer_type (cmp_bits, true);
1061 110571 : if (this_type
1062 110571 : && can_produce_all_loop_masks_p (loop_vinfo, this_type))
1063 : {
1064 : /* Although we could stop as soon as we find a valid mode,
1065 : there are at least two reasons why that's not always the
1066 : best choice:
1067 :
1068 : - An IV that's Pmode or wider is more likely to be reusable
1069 : in address calculations than an IV that's narrower than
1070 : Pmode.
1071 :
1072 : - Doing the comparison in IV_PRECISION or wider allows
1073 : a natural 0-based IV, whereas using a narrower comparison
1074 : type requires mitigations against wrap-around.
1075 :
1076 : Conversely, if the IV limit is variable, doing the comparison
1077 : in a wider type than the original type can introduce
1078 : unnecessary extensions, so picking the widest valid mode
1079 : is not always a good choice either.
1080 :
1081 : Here we prefer the first IV type that's Pmode or wider,
1082 : and the first comparison type that's IV_PRECISION or wider.
1083 : (The comparison type must be no wider than the IV type,
1084 : to avoid extensions in the vector loop.)
1085 :
1086 : ??? We might want to try continuing beyond Pmode for ILP32
1087 : targets if CMP_BITS < IV_PRECISION. */
1088 0 : iv_type = this_type;
1089 0 : if (!cmp_type || iv_precision > TYPE_PRECISION (cmp_type))
1090 : cmp_type = this_type;
1091 0 : if (cmp_bits >= GET_MODE_BITSIZE (Pmode))
1092 : break;
1093 : }
1094 : }
1095 : }
1096 :
1097 23571 : if (!cmp_type)
1098 : {
1099 23571 : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.release ();
1100 23571 : return false;
1101 : }
1102 :
1103 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
1104 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1105 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_while_ult;
1106 0 : return true;
1107 23571 : }
1108 :
1109 : /* Each statement in LOOP_VINFO can be masked where necessary. Check
1110 : whether we can actually generate AVX512 style masks. Return true if so,
1111 : storing the type of the scalar IV in LOOP_VINFO_RGROUP_IV_TYPE. */
1112 :
1113 : static bool
1114 23571 : vect_verify_full_masking_avx512 (loop_vec_info loop_vinfo)
1115 : {
1116 : /* Produce differently organized rgc_vec and differently check
1117 : we can produce masks. */
1118 :
1119 : /* Use a normal loop if there are no statements that need masking.
1120 : This only happens in rare degenerate cases: it means that the loop
1121 : has no loads, no stores, and no live-out values. */
1122 23571 : if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
1123 : return false;
1124 :
1125 : /* For the decrementing IV we need to represent all values in
1126 : [0, niter + niter_skip] where niter_skip is the elements we
1127 : skip in the first iteration for prologue peeling. */
1128 23571 : tree iv_type = NULL_TREE;
1129 23571 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
1130 23571 : unsigned int iv_precision = UINT_MAX;
1131 23571 : if (iv_limit != -1)
1132 23571 : iv_precision = wi::min_precision (iv_limit, UNSIGNED);
1133 :
1134 : /* First compute the type for the IV we use to track the remaining
1135 : scalar iterations. */
1136 23571 : opt_scalar_int_mode cmp_mode_iter;
1137 30833 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1138 : {
1139 30833 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1140 30833 : if (cmp_bits >= iv_precision
1141 30833 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1142 : {
1143 23571 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
1144 23571 : if (iv_type)
1145 : break;
1146 : }
1147 : }
1148 23571 : if (!iv_type)
1149 : return false;
1150 :
1151 : /* Produce the rgroup controls. */
1152 58240 : for (auto const &mask : LOOP_VINFO_MASKS (loop_vinfo).mask_set)
1153 : {
1154 34669 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1155 34669 : tree vectype = mask.first;
1156 34669 : unsigned nvectors = mask.second;
1157 :
1158 : /* The number of scalars per iteration and the number of vectors are
1159 : both compile-time constants. */
1160 34669 : unsigned int nscalars_per_iter
1161 34669 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
1162 34669 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
1163 :
1164 : /* We index the rgroup_controls vector with nscalars_per_iter
1165 : which we keep constant and instead have a varying nvectors,
1166 : remembering the vector mask with the fewest nV. */
1167 45767 : if (masks->rgc_vec.length () < nscalars_per_iter)
1168 23628 : masks->rgc_vec.safe_grow_cleared (nscalars_per_iter, true);
1169 34669 : rgroup_controls *rgm = &(*masks).rgc_vec[nscalars_per_iter - 1];
1170 :
1171 34669 : if (!rgm->type || rgm->factor > nvectors)
1172 : {
1173 25481 : rgm->type = truth_type_for (vectype);
1174 25481 : rgm->compare_type = NULL_TREE;
1175 25481 : rgm->max_nscalars_per_iter = nscalars_per_iter;
1176 25481 : rgm->factor = nvectors;
1177 25481 : rgm->bias_adjusted_ctrl = NULL_TREE;
1178 : }
1179 : }
1180 :
1181 : /* There is no fixed compare type we are going to use but we have to
1182 : be able to get at one for each mask group. */
1183 23571 : unsigned int min_ni_width
1184 23571 : = wi::min_precision (vect_max_vf (loop_vinfo), UNSIGNED);
1185 :
1186 23571 : bool ok = true;
1187 89239 : for (auto &rgc : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
1188 : {
1189 24640 : tree mask_type = rgc.type;
1190 24640 : if (!mask_type)
1191 990 : continue;
1192 :
1193 : /* For now vect_get_loop_mask only supports integer mode masks
1194 : when we need to split it. */
1195 23650 : if (GET_MODE_CLASS (TYPE_MODE (mask_type)) != MODE_INT
1196 23650 : || TYPE_PRECISION (TREE_TYPE (mask_type)) != 1)
1197 : {
1198 : ok = false;
1199 : break;
1200 : }
1201 :
1202 : /* If iv_type is usable as compare type use that - we can elide the
1203 : saturation in that case. */
1204 17540 : if (TYPE_PRECISION (iv_type) >= min_ni_width)
1205 : {
1206 17540 : tree cmp_vectype
1207 17540 : = build_vector_type (iv_type, TYPE_VECTOR_SUBPARTS (mask_type));
1208 17540 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1209 5978 : rgc.compare_type = cmp_vectype;
1210 : }
1211 17540 : if (!rgc.compare_type)
1212 33257 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
1213 : {
1214 33253 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode_iter.require ());
1215 33253 : if (cmp_bits >= min_ni_width
1216 33253 : && targetm.scalar_mode_supported_p (cmp_mode_iter.require ()))
1217 : {
1218 33241 : tree cmp_type = build_nonstandard_integer_type (cmp_bits, true);
1219 33241 : if (!cmp_type)
1220 0 : continue;
1221 :
1222 : /* Check whether we can produce the mask with cmp_type. */
1223 33241 : tree cmp_vectype
1224 33241 : = build_vector_type (cmp_type, TYPE_VECTOR_SUBPARTS (mask_type));
1225 33241 : if (expand_vec_cmp_expr_p (cmp_vectype, mask_type, LT_EXPR))
1226 : {
1227 11558 : rgc.compare_type = cmp_vectype;
1228 11558 : break;
1229 : }
1230 : }
1231 : }
1232 17540 : if (!rgc.compare_type)
1233 : {
1234 : ok = false;
1235 : break;
1236 : }
1237 : }
1238 23571 : if (!ok)
1239 : {
1240 6114 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
1241 6114 : return false;
1242 : }
1243 :
1244 17457 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = error_mark_node;
1245 17457 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1246 17457 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_avx512;
1247 17457 : return true;
1248 23571 : }
1249 :
1250 : /* Check whether we can use vector access with length based on precision
1251 : comparison. So far, to keep it simple, we only allow the case that the
1252 : precision of the target supported length is larger than the precision
1253 : required by loop niters. */
1254 :
1255 : static bool
1256 6 : vect_verify_loop_lens (loop_vec_info loop_vinfo)
1257 : {
1258 6 : if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
1259 : return false;
1260 :
1261 0 : if (!VECTOR_MODE_P (loop_vinfo->vector_mode))
1262 : return false;
1263 :
1264 0 : machine_mode len_load_mode, len_store_mode;
1265 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
1266 0 : .exists (&len_load_mode))
1267 0 : return false;
1268 0 : if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
1269 0 : .exists (&len_store_mode))
1270 0 : return false;
1271 :
1272 0 : signed char partial_load_bias = internal_len_load_store_bias
1273 0 : (IFN_LEN_LOAD, len_load_mode);
1274 :
1275 0 : signed char partial_store_bias = internal_len_load_store_bias
1276 0 : (IFN_LEN_STORE, len_store_mode);
1277 :
1278 0 : gcc_assert (partial_load_bias == partial_store_bias);
1279 :
1280 0 : if (partial_load_bias == VECT_PARTIAL_BIAS_UNSUPPORTED)
1281 : return false;
1282 :
1283 : /* If the backend requires a bias of -1 for LEN_LOAD, we must not emit
1284 : len_loads with a length of zero. In order to avoid that we prohibit
1285 : more than one loop length here. */
1286 0 : if (partial_load_bias == -1
1287 0 : && LOOP_VINFO_LENS (loop_vinfo).length () > 1)
1288 : return false;
1289 :
1290 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) = partial_load_bias;
1291 :
1292 0 : unsigned int max_nitems_per_iter = 1;
1293 0 : unsigned int i;
1294 0 : rgroup_controls *rgl;
1295 : /* Find the maximum number of items per iteration for every rgroup. */
1296 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), i, rgl)
1297 : {
1298 0 : unsigned nitems_per_iter = rgl->max_nscalars_per_iter * rgl->factor;
1299 0 : max_nitems_per_iter = MAX (max_nitems_per_iter, nitems_per_iter);
1300 : }
1301 :
1302 : /* Work out how many bits we need to represent the length limit. */
1303 0 : unsigned int min_ni_prec
1304 0 : = vect_min_prec_for_max_niters (loop_vinfo, max_nitems_per_iter);
1305 :
1306 : /* Now use the maximum of below precisions for one suitable IV type:
1307 : - the IV's natural precision
1308 : - the precision needed to hold: the maximum number of scalar
1309 : iterations multiplied by the scale factor (min_ni_prec above)
1310 : - the Pmode precision
1311 :
1312 : If min_ni_prec is less than the precision of the current niters,
1313 : we prefer to still use the niters type. Prefer to use Pmode and
1314 : wider IV to avoid narrow conversions. */
1315 :
1316 0 : unsigned int ni_prec
1317 0 : = TYPE_PRECISION (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)));
1318 0 : min_ni_prec = MAX (min_ni_prec, ni_prec);
1319 0 : min_ni_prec = MAX (min_ni_prec, GET_MODE_BITSIZE (Pmode));
1320 :
1321 0 : tree iv_type = NULL_TREE;
1322 0 : opt_scalar_int_mode tmode_iter;
1323 0 : FOR_EACH_MODE_IN_CLASS (tmode_iter, MODE_INT)
1324 : {
1325 0 : scalar_mode tmode = tmode_iter.require ();
1326 0 : unsigned int tbits = GET_MODE_BITSIZE (tmode);
1327 :
1328 : /* ??? Do we really want to construct one IV whose precision exceeds
1329 : BITS_PER_WORD? */
1330 0 : if (tbits > BITS_PER_WORD)
1331 : break;
1332 :
1333 : /* Find the first available standard integral type. */
1334 0 : if (tbits >= min_ni_prec && targetm.scalar_mode_supported_p (tmode))
1335 : {
1336 0 : iv_type = build_nonstandard_integer_type (tbits, true);
1337 0 : break;
1338 : }
1339 : }
1340 :
1341 0 : if (!iv_type)
1342 : {
1343 0 : if (dump_enabled_p ())
1344 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1345 : "can't vectorize with length-based partial vectors"
1346 : " because there is no suitable iv type.\n");
1347 : return false;
1348 : }
1349 :
1350 0 : LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = iv_type;
1351 0 : LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
1352 0 : LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_len;
1353 :
1354 0 : return true;
1355 : }
1356 :
1357 : /* Calculate the cost of one scalar iteration of the loop. */
1358 : static void
1359 377747 : vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
1360 : {
1361 377747 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1362 377747 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
1363 377747 : int nbbs = loop->num_nodes, factor;
1364 377747 : int innerloop_iters, i;
1365 :
1366 377747 : DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
1367 :
1368 : /* Gather costs for statements in the scalar loop. */
1369 :
1370 : /* FORNOW. */
1371 377747 : innerloop_iters = 1;
1372 377747 : if (loop->inner)
1373 1602 : innerloop_iters = LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
1374 :
1375 1302234 : for (i = 0; i < nbbs; i++)
1376 : {
1377 924487 : gimple_stmt_iterator si;
1378 924487 : basic_block bb = bbs[i];
1379 :
1380 924487 : if (bb->loop_father == loop->inner)
1381 : factor = innerloop_iters;
1382 : else
1383 921283 : factor = 1;
1384 :
1385 7622908 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
1386 : {
1387 5773934 : gimple *stmt = gsi_stmt (si);
1388 5773934 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
1389 :
1390 5773934 : if (!is_gimple_assign (stmt)
1391 : && !is_gimple_call (stmt)
1392 : && !is_a<gcond *> (stmt))
1393 2129852 : continue;
1394 :
1395 : /* Skip stmts that are not vectorized inside the loop. */
1396 3644082 : stmt_vec_info vstmt_info = vect_stmt_to_vectorize (stmt_info);
1397 3644082 : if (!STMT_VINFO_RELEVANT_P (vstmt_info)
1398 1801644 : && (!STMT_VINFO_LIVE_P (vstmt_info)
1399 51 : || !VECTORIZABLE_CYCLE_DEF
1400 : (STMT_VINFO_DEF_TYPE (vstmt_info))))
1401 1801644 : continue;
1402 :
1403 1842438 : vect_cost_for_stmt kind;
1404 1842438 : if (STMT_VINFO_DATA_REF (stmt_info))
1405 : {
1406 888524 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
1407 : kind = scalar_load;
1408 : else
1409 330527 : kind = scalar_store;
1410 : }
1411 953914 : else if (vect_nop_conversion_p (stmt_info))
1412 54187 : continue;
1413 : else
1414 : kind = scalar_stmt;
1415 :
1416 : /* We are using vect_prologue here to avoid scaling twice
1417 : by the inner loop factor. */
1418 1788251 : record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
1419 : factor, kind, stmt_info, 0, vect_body);
1420 : }
1421 : }
1422 :
1423 : /* Now accumulate cost. */
1424 377747 : loop_vinfo->scalar_costs = init_cost (loop_vinfo, true);
1425 377747 : add_stmt_costs (loop_vinfo->scalar_costs,
1426 : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo));
1427 377747 : loop_vinfo->scalar_costs->finish_cost (nullptr);
1428 377747 : }
1429 :
1430 : /* Function vect_analyze_loop_form.
1431 :
1432 : Verify that certain CFG restrictions hold, including:
1433 : - the loop has a pre-header
1434 : - the loop has a single entry
1435 : - nested loops can have only a single exit.
1436 : - the loop exit condition is simple enough
1437 : - the number of iterations can be analyzed, i.e, a countable loop. The
1438 : niter could be analyzed under some assumptions. */
1439 :
1440 : opt_result
1441 470395 : vect_analyze_loop_form (class loop *loop, gimple *loop_vectorized_call,
1442 : vect_loop_form_info *info)
1443 : {
1444 470395 : DUMP_VECT_SCOPE ("vect_analyze_loop_form");
1445 :
1446 470395 : edge exit_e = vec_init_loop_exit_info (loop);
1447 470395 : if (!exit_e)
1448 30900 : return opt_result::failure_at (vect_location,
1449 : "not vectorized:"
1450 : " Infinite loop detected.\n");
1451 439495 : if (loop_vectorized_call)
1452 : {
1453 25319 : tree arg = gimple_call_arg (loop_vectorized_call, 1);
1454 25319 : class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
1455 25319 : edge scalar_exit_e = vec_init_loop_exit_info (scalar_loop);
1456 25319 : if (!scalar_exit_e)
1457 0 : return opt_result::failure_at (vect_location,
1458 : "not vectorized:"
1459 : " could not determine main exit from"
1460 : " loop with multiple exits.\n");
1461 : }
1462 :
1463 439495 : info->loop_exit = exit_e;
1464 439495 : if (dump_enabled_p ())
1465 16416 : dump_printf_loc (MSG_NOTE, vect_location,
1466 : "using as main loop exit: %d -> %d [AUX: %p]\n",
1467 16416 : exit_e->src->index, exit_e->dest->index, exit_e->aux);
1468 :
1469 : /* Check if we have any control flow that doesn't leave the loop. */
1470 439495 : basic_block *bbs = get_loop_body (loop);
1471 1881927 : for (unsigned i = 0; i < loop->num_nodes; i++)
1472 1122417 : if (EDGE_COUNT (bbs[i]->succs) != 1
1473 1122417 : && (EDGE_COUNT (bbs[i]->succs) != 2
1474 671765 : || !loop_exits_from_bb_p (bbs[i]->loop_father, bbs[i])))
1475 : {
1476 119480 : free (bbs);
1477 119480 : return opt_result::failure_at (vect_location,
1478 : "not vectorized:"
1479 : " unsupported control flow in loop.\n");
1480 : }
1481 :
1482 : /* Check if we have any control flow that doesn't leave the loop. */
1483 321115 : bool has_phi = false;
1484 321115 : for (unsigned i = 0; i < loop->num_nodes; i++)
1485 320657 : if (!gimple_seq_empty_p (phi_nodes (bbs[i])))
1486 : {
1487 : has_phi = true;
1488 : break;
1489 : }
1490 320015 : if (!has_phi)
1491 458 : return opt_result::failure_at (vect_location,
1492 : "not vectorized:"
1493 : " no scalar evolution detected in loop.\n");
1494 :
1495 319557 : free (bbs);
1496 :
1497 : /* Different restrictions apply when we are considering an inner-most loop,
1498 : vs. an outer (nested) loop.
1499 : (FORNOW. May want to relax some of these restrictions in the future). */
1500 :
1501 319557 : info->inner_loop_cond = NULL;
1502 319557 : if (!loop->inner)
1503 : {
1504 : /* Inner-most loop. */
1505 :
1506 300444 : if (empty_block_p (loop->header))
1507 0 : return opt_result::failure_at (vect_location,
1508 : "not vectorized: empty loop.\n");
1509 : }
1510 : else
1511 : {
1512 19113 : class loop *innerloop = loop->inner;
1513 19113 : edge entryedge;
1514 :
1515 : /* Nested loop. We currently require that the loop is doubly-nested,
1516 : contains a single inner loop with a single exit to the block
1517 : with the single exit condition in the outer loop.
1518 : Vectorizable outer-loops look like this:
1519 :
1520 : (pre-header)
1521 : |
1522 : header <---+
1523 : | |
1524 : inner-loop |
1525 : | |
1526 : tail ------+
1527 : |
1528 : (exit-bb)
1529 :
1530 : The inner-loop also has the properties expected of inner-most loops
1531 : as described above. */
1532 :
1533 19113 : if ((loop->inner)->inner || (loop->inner)->next)
1534 3056 : return opt_result::failure_at (vect_location,
1535 : "not vectorized:"
1536 : " multiple nested loops.\n");
1537 :
1538 16057 : entryedge = loop_preheader_edge (innerloop);
1539 16057 : if (entryedge->src != loop->header
1540 15516 : || !single_exit (innerloop)
1541 27590 : || single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
1542 4833 : return opt_result::failure_at (vect_location,
1543 : "not vectorized:"
1544 : " unsupported outerloop form.\n");
1545 :
1546 : /* Analyze the inner-loop. */
1547 11224 : vect_loop_form_info inner;
1548 11224 : opt_result res = vect_analyze_loop_form (loop->inner, NULL, &inner);
1549 11224 : if (!res)
1550 : {
1551 417 : if (dump_enabled_p ())
1552 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1553 : "not vectorized: Bad inner loop.\n");
1554 417 : return res;
1555 : }
1556 :
1557 : /* Don't support analyzing niter under assumptions for inner
1558 : loop. */
1559 10807 : if (!integer_onep (inner.assumptions))
1560 263 : return opt_result::failure_at (vect_location,
1561 : "not vectorized: Bad inner loop.\n");
1562 :
1563 10544 : if (inner.number_of_iterations == chrec_dont_know
1564 10544 : || !expr_invariant_in_loop_p (loop, inner.number_of_iterations))
1565 1853 : return opt_result::failure_at (vect_location,
1566 : "not vectorized: inner-loop count not"
1567 : " invariant.\n");
1568 :
1569 8691 : if (dump_enabled_p ())
1570 1050 : dump_printf_loc (MSG_NOTE, vect_location,
1571 : "Considering outer-loop vectorization.\n");
1572 8691 : info->inner_loop_cond = inner.conds[0];
1573 11224 : }
1574 :
1575 309135 : if (EDGE_COUNT (loop->header->preds) != 2)
1576 0 : return opt_result::failure_at (vect_location,
1577 : "not vectorized:"
1578 : " too many incoming edges.\n");
1579 :
1580 : /* We assume that the latch is empty. */
1581 309135 : basic_block latch = loop->latch;
1582 309135 : do
1583 : {
1584 309135 : if (!empty_block_p (latch)
1585 309135 : || !gimple_seq_empty_p (phi_nodes (latch)))
1586 20369 : return opt_result::failure_at (vect_location,
1587 : "not vectorized: latch block not "
1588 : "empty.\n");
1589 288766 : latch = single_pred (latch);
1590 : }
1591 577532 : while (single_succ_p (latch));
1592 :
1593 : /* Make sure there is no abnormal exit. */
1594 288766 : auto_vec<edge> exits = get_loop_exit_edges (loop);
1595 1282904 : for (edge e : exits)
1596 : {
1597 416639 : if (e->flags & EDGE_ABNORMAL)
1598 33 : return opt_result::failure_at (vect_location,
1599 : "not vectorized:"
1600 : " abnormal loop exit edge.\n");
1601 : }
1602 :
1603 288733 : info->conds
1604 288733 : = vect_get_loop_niters (loop, exit_e, &info->assumptions,
1605 : &info->number_of_iterations,
1606 288733 : &info->number_of_iterationsm1);
1607 288733 : if (info->conds.is_empty ())
1608 34 : return opt_result::failure_at
1609 34 : (vect_location,
1610 : "not vectorized: complicated exit condition.\n");
1611 :
1612 : /* Determine what the primary and alternate exit conds are. */
1613 705271 : for (unsigned i = 0; i < info->conds.length (); i++)
1614 : {
1615 416572 : gcond *cond = info->conds[i];
1616 416572 : if (exit_e->src == gimple_bb (cond))
1617 288699 : std::swap (info->conds[0], info->conds[i]);
1618 : }
1619 :
1620 288699 : if (chrec_contains_undetermined (info->number_of_iterations))
1621 : {
1622 61316 : if (dump_enabled_p ())
1623 257 : dump_printf_loc (MSG_NOTE, vect_location,
1624 : "Loop being analyzed as uncounted.\n");
1625 61316 : if (loop->inner)
1626 563 : return opt_result::failure_at
1627 563 : (vect_location,
1628 : "not vectorized: outer loop vectorization of uncounted loops"
1629 : " is unsupported.\n");
1630 60753 : return opt_result::success ();
1631 : }
1632 :
1633 227383 : if (integer_zerop (info->assumptions))
1634 4 : return opt_result::failure_at
1635 4 : (info->conds[0],
1636 : "not vectorized: number of iterations cannot be computed.\n");
1637 :
1638 227379 : if (integer_zerop (info->number_of_iterations))
1639 12 : return opt_result::failure_at
1640 12 : (info->conds[0],
1641 : "not vectorized: number of iterations = 0.\n");
1642 :
1643 227367 : if (!(tree_fits_shwi_p (info->number_of_iterations)
1644 125404 : && tree_to_shwi (info->number_of_iterations) > 0))
1645 : {
1646 101963 : if (dump_enabled_p ())
1647 : {
1648 2655 : dump_printf_loc (MSG_NOTE, vect_location,
1649 : "Symbolic number of iterations is ");
1650 2655 : dump_generic_expr (MSG_NOTE, TDF_DETAILS, info->number_of_iterations);
1651 2655 : dump_printf (MSG_NOTE, "\n");
1652 : }
1653 : }
1654 :
1655 227367 : if (!integer_onep (info->assumptions))
1656 : {
1657 9171 : if (dump_enabled_p ())
1658 : {
1659 78 : dump_printf_loc (MSG_NOTE, vect_location,
1660 : "Loop to be versioned with niter assumption ");
1661 78 : dump_generic_expr (MSG_NOTE, TDF_SLIM, info->assumptions);
1662 78 : dump_printf (MSG_NOTE, "\n");
1663 : }
1664 : }
1665 :
1666 227367 : return opt_result::success ();
1667 288766 : }
1668 :
1669 : /* Create a loop_vec_info for LOOP with SHARED and the
1670 : vect_analyze_loop_form result. */
1671 :
1672 : loop_vec_info
1673 598181 : vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
1674 : const vect_loop_form_info *info,
1675 : loop_vec_info orig_loop_info)
1676 : {
1677 598181 : loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared);
1678 598181 : LOOP_VINFO_NITERSM1 (loop_vinfo) = info->number_of_iterationsm1;
1679 598181 : LOOP_VINFO_NITERS (loop_vinfo) = info->number_of_iterations;
1680 598181 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = info->number_of_iterations;
1681 598181 : LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = orig_loop_info;
1682 598181 : if (orig_loop_info && LOOP_VINFO_EPILOGUE_P (orig_loop_info))
1683 404 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo)
1684 404 : = LOOP_VINFO_MAIN_LOOP_INFO (orig_loop_info);
1685 : else
1686 597777 : LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo) = orig_loop_info;
1687 : /* Also record the assumptions for versioning. */
1688 598181 : if (!integer_onep (info->assumptions) && !orig_loop_info)
1689 20932 : LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = info->assumptions;
1690 :
1691 2658486 : for (gcond *cond : info->conds)
1692 : {
1693 863943 : stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
1694 : /* Mark the statement as a condition. */
1695 863943 : STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
1696 : }
1697 :
1698 598181 : unsigned cond_id = 0;
1699 598181 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
1700 509206 : LOOP_VINFO_LOOP_IV_COND (loop_vinfo) = info->conds[cond_id++];
1701 :
1702 952918 : for (; cond_id < info->conds.length (); cond_id ++)
1703 354737 : LOOP_VINFO_LOOP_CONDS (loop_vinfo).safe_push (info->conds[cond_id]);
1704 :
1705 598181 : LOOP_VINFO_MAIN_EXIT (loop_vinfo) = info->loop_exit;
1706 :
1707 : /* Check to see if we're vectorizing multiple exits. */
1708 598181 : LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
1709 598181 : = !LOOP_VINFO_LOOP_CONDS (loop_vinfo).is_empty ();
1710 :
1711 : /* At the moment we can't support no epilogs for multiple exits, result of
1712 : the first compare should be masked by that of the second. We can only
1713 : allow it if the early exits have the same live values. for differing
1714 : values we have to calculate a third mask to disambiguate. */
1715 598181 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo)
1716 598181 : = LOOP_VINFO_LOOP_CONDS (loop_vinfo).length () > 1;
1717 :
1718 598181 : if (info->inner_loop_cond)
1719 : {
1720 : /* If we have an estimate on the number of iterations of the inner
1721 : loop use that to limit the scale for costing, otherwise use
1722 : --param vect-inner-loop-cost-factor literally. */
1723 9107 : widest_int nit;
1724 9107 : if (estimated_stmt_executions (loop->inner, &nit))
1725 7792 : LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo)
1726 7792 : = wi::smin (nit, param_vect_inner_loop_cost_factor).to_uhwi ();
1727 9107 : }
1728 :
1729 598181 : return loop_vinfo;
1730 : }
1731 :
1732 :
1733 :
1734 : /* Return true if we know that the iteration count is smaller than the
1735 : vectorization factor. Return false if it isn't, or if we can't be sure
1736 : either way. */
1737 :
1738 : static bool
1739 156825 : vect_known_niters_smaller_than_vf (loop_vec_info loop_vinfo)
1740 : {
1741 156825 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1742 :
1743 156825 : HOST_WIDE_INT max_niter;
1744 156825 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1745 80731 : max_niter = LOOP_VINFO_INT_NITERS (loop_vinfo);
1746 : else
1747 76094 : max_niter = max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo));
1748 :
1749 156825 : if (max_niter != -1 && (unsigned HOST_WIDE_INT) max_niter < assumed_vf)
1750 11090 : return true;
1751 :
1752 : return false;
1753 : }
1754 :
1755 : /* Analyze the cost of the loop described by LOOP_VINFO. Decide if it
1756 : is worthwhile to vectorize. Return 1 if definitely yes, 0 if
1757 : definitely no, or -1 if it's worth retrying. */
1758 :
1759 : static int
1760 156838 : vect_analyze_loop_costing (loop_vec_info loop_vinfo,
1761 : unsigned *suggested_unroll_factor)
1762 : {
1763 156838 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1764 156838 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
1765 :
1766 : /* Only loops that can handle partially-populated vectors can have iteration
1767 : counts less than the vectorization factor. */
1768 156838 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
1769 156838 : && vect_known_niters_smaller_than_vf (loop_vinfo))
1770 : {
1771 11080 : if (dump_enabled_p ())
1772 247 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1773 : "not vectorized: iteration count smaller than "
1774 : "vectorization factor.\n");
1775 : return 0;
1776 : }
1777 :
1778 : /* If we know the number of iterations we can do better, for the
1779 : epilogue we can also decide whether the main loop leaves us
1780 : with enough iterations, preferring a smaller vector epilog then
1781 : also possibly used for the case we skip the vector loop. */
1782 145758 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
1783 : {
1784 70935 : widest_int scalar_niters
1785 70935 : = wi::to_widest (LOOP_VINFO_NITERSM1 (loop_vinfo)) + 1;
1786 70935 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1787 : {
1788 2810 : loop_vec_info orig_loop_vinfo
1789 : = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
1790 2810 : loop_vec_info main_loop_vinfo
1791 : = LOOP_VINFO_MAIN_LOOP_INFO (loop_vinfo);
1792 2810 : unsigned lowest_vf
1793 2810 : = constant_lower_bound (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo));
1794 2810 : int prolog_peeling = 0;
1795 2810 : if (!vect_use_loop_mask_for_alignment_p (main_loop_vinfo))
1796 2810 : prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (main_loop_vinfo);
1797 2810 : if (prolog_peeling >= 0
1798 2810 : && known_eq (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
1799 : lowest_vf))
1800 : {
1801 5610 : unsigned gap
1802 2805 : = LOOP_VINFO_PEELING_FOR_GAPS (main_loop_vinfo) ? 1 : 0;
1803 5610 : scalar_niters = ((scalar_niters - gap - prolog_peeling)
1804 5610 : % lowest_vf + gap);
1805 : }
1806 : }
1807 : /* Reject vectorizing for a single scalar iteration, even if
1808 : we could in principle implement that using partial vectors.
1809 : But allow such vectorization if VF == 1 in case we do not
1810 : need to peel for gaps (if we need, avoid vectorization for
1811 : reasons of code footprint). */
1812 70935 : unsigned peeling_gap = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo);
1813 70935 : if (scalar_niters <= peeling_gap + 1
1814 70935 : && (assumed_vf > 1 || peeling_gap != 0))
1815 : {
1816 670 : if (dump_enabled_p ())
1817 162 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1818 : "not vectorized: loop only has a single "
1819 : "scalar iteration.\n");
1820 : return 0;
1821 : }
1822 :
1823 70265 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
1824 : {
1825 : /* Check that the loop processes at least one full vector. */
1826 70252 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1827 70252 : if (known_lt (scalar_niters, vf))
1828 : {
1829 401 : if (dump_enabled_p ())
1830 347 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1831 : "loop does not have enough iterations "
1832 : "to support vectorization.\n");
1833 441 : return 0;
1834 : }
1835 :
1836 : /* If we need to peel an extra epilogue iteration to handle data
1837 : accesses with gaps, check that there are enough scalar iterations
1838 : available.
1839 :
1840 : The check above is redundant with this one when peeling for gaps,
1841 : but the distinction is useful for diagnostics. */
1842 69851 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
1843 70172 : && known_le (scalar_niters, vf))
1844 : {
1845 40 : if (dump_enabled_p ())
1846 9 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1847 : "loop does not have enough iterations "
1848 : "to support peeling for gaps.\n");
1849 : return 0;
1850 : }
1851 : }
1852 70935 : }
1853 :
1854 : /* If using the "very cheap" model. reject cases in which we'd keep
1855 : a copy of the scalar code (even if we might be able to vectorize it). */
1856 144647 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1857 144647 : && (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1858 76544 : || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
1859 : {
1860 721 : if (dump_enabled_p ())
1861 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1862 : "some scalar iterations would need to be peeled\n");
1863 : return 0;
1864 : }
1865 :
1866 143926 : int min_profitable_iters, min_profitable_estimate;
1867 143926 : vect_estimate_min_profitable_iters (loop_vinfo, &min_profitable_iters,
1868 : &min_profitable_estimate,
1869 : suggested_unroll_factor);
1870 :
1871 143926 : if (min_profitable_iters < 0)
1872 : {
1873 24493 : if (dump_enabled_p ())
1874 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1875 : "not vectorized: vectorization not profitable.\n");
1876 24493 : if (dump_enabled_p ())
1877 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1878 : "not vectorized: vector version will never be "
1879 : "profitable.\n");
1880 : return -1;
1881 : }
1882 :
1883 119433 : int min_scalar_loop_bound = (param_min_vect_loop_bound
1884 119433 : * assumed_vf);
1885 :
1886 : /* Use the cost model only if it is more conservative than user specified
1887 : threshold. */
1888 119433 : unsigned int th = (unsigned) MAX (min_scalar_loop_bound,
1889 : min_profitable_iters);
1890 :
1891 119433 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = th;
1892 :
1893 64427 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1894 183860 : && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
1895 : {
1896 564 : if (dump_enabled_p ())
1897 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1898 : "not vectorized: vectorization not profitable.\n");
1899 564 : if (dump_enabled_p ())
1900 2 : dump_printf_loc (MSG_NOTE, vect_location,
1901 : "not vectorized: iteration count smaller than user "
1902 : "specified loop bound parameter or minimum profitable "
1903 : "iterations (whichever is more conservative).\n");
1904 : return 0;
1905 : }
1906 :
1907 : /* The static profitablity threshold min_profitable_estimate includes
1908 : the cost of having to check at runtime whether the scalar loop
1909 : should be used instead. If it turns out that we don't need or want
1910 : such a check, the threshold we should use for the static estimate
1911 : is simply the point at which the vector loop becomes more profitable
1912 : than the scalar loop. */
1913 118869 : if (min_profitable_estimate > min_profitable_iters
1914 27104 : && !LOOP_REQUIRES_VERSIONING (loop_vinfo)
1915 26528 : && !LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
1916 642 : && !LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1917 119511 : && !vect_apply_runtime_profitability_check_p (loop_vinfo))
1918 : {
1919 14 : if (dump_enabled_p ())
1920 8 : dump_printf_loc (MSG_NOTE, vect_location, "no need for a runtime"
1921 : " choice between the scalar and vector loops\n");
1922 14 : min_profitable_estimate = min_profitable_iters;
1923 : }
1924 :
1925 : /* If the vector loop needs multiple iterations to be beneficial then
1926 : things are probably too close to call, and the conservative thing
1927 : would be to stick with the scalar code. */
1928 118869 : if (loop_cost_model (loop) == VECT_COST_MODEL_VERY_CHEAP
1929 61097 : && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1930 142277 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1931 : {
1932 14728 : if (dump_enabled_p ())
1933 225 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1934 : "one iteration of the vector loop would be"
1935 : " more expensive than the equivalent number of"
1936 : " iterations of the scalar loop\n");
1937 : return 0;
1938 : }
1939 :
1940 104141 : HOST_WIDE_INT estimated_niter;
1941 :
1942 : /* If we are vectorizing an epilogue then we know the maximum number of
1943 : scalar iterations it will cover is at least one lower than the
1944 : vectorization factor of the main loop. */
1945 104141 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1946 12304 : estimated_niter
1947 12304 : = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
1948 : else
1949 : {
1950 91837 : estimated_niter = estimated_stmt_executions_int (loop);
1951 91837 : if (estimated_niter == -1)
1952 30139 : estimated_niter = likely_max_stmt_executions_int (loop);
1953 : }
1954 42443 : if (estimated_niter != -1
1955 101204 : && ((unsigned HOST_WIDE_INT) estimated_niter
1956 101204 : < MAX (th, (unsigned) min_profitable_estimate)))
1957 : {
1958 4505 : if (dump_enabled_p ())
1959 34 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1960 : "not vectorized: estimated iteration count too "
1961 : "small.\n");
1962 4505 : if (dump_enabled_p ())
1963 34 : dump_printf_loc (MSG_NOTE, vect_location,
1964 : "not vectorized: estimated iteration count smaller "
1965 : "than specified loop bound parameter or minimum "
1966 : "profitable iterations (whichever is more "
1967 : "conservative).\n");
1968 : return -1;
1969 : }
1970 :
1971 : /* As we cannot use a runtime check to gate profitability for uncounted
1972 : loops require either an estimate or if none, at least a profitable
1973 : vectorization within the first vector iteration (that condition
1974 : will practically never be true due to the required epilog and
1975 : likely alignment prologue). */
1976 99636 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
1977 163 : && estimated_niter == -1
1978 99772 : && min_profitable_estimate > (int) vect_vf_for_cost (loop_vinfo))
1979 : {
1980 120 : if (dump_enabled_p ())
1981 2 : dump_printf_loc (MSG_NOTE, vect_location,
1982 : "not vectorized: no loop iteration estimate on the "
1983 : "uncounted loop and not trivially profitable.\n");
1984 : return -1;
1985 : }
1986 :
1987 : return 1;
1988 : }
1989 :
1990 : /* Gather data references in LOOP with body BBS and store them into
1991 : *DATAREFS. */
1992 :
1993 : static opt_result
1994 286449 : vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
1995 : vec<data_reference_p> *datarefs)
1996 : {
1997 858526 : for (unsigned i = 0; i < loop->num_nodes; i++)
1998 1271724 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
1999 5528878 : !gsi_end_p (gsi); gsi_next (&gsi))
2000 : {
2001 4956801 : gimple *stmt = gsi_stmt (gsi);
2002 4956801 : if (is_gimple_debug (stmt))
2003 2388339 : continue;
2004 2568592 : opt_result res = vect_find_stmt_data_reference (loop, stmt, datarefs,
2005 : NULL, 0);
2006 2568592 : if (!res)
2007 : {
2008 63915 : if (is_gimple_call (stmt) && loop->safelen)
2009 : {
2010 406 : tree fndecl = gimple_call_fndecl (stmt), op;
2011 406 : if (fndecl == NULL_TREE
2012 406 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
2013 : {
2014 0 : fndecl = gimple_call_arg (stmt, 0);
2015 0 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
2016 0 : fndecl = TREE_OPERAND (fndecl, 0);
2017 0 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
2018 : }
2019 406 : if (fndecl != NULL_TREE)
2020 : {
2021 369 : cgraph_node *node = cgraph_node::get (fndecl);
2022 369 : if (node != NULL && node->simd_clones != NULL)
2023 : {
2024 131 : unsigned int j, n = gimple_call_num_args (stmt);
2025 545 : for (j = 0; j < n; j++)
2026 : {
2027 284 : op = gimple_call_arg (stmt, j);
2028 284 : if (DECL_P (op)
2029 284 : || (REFERENCE_CLASS_P (op)
2030 0 : && get_base_address (op)))
2031 : break;
2032 : }
2033 131 : op = gimple_call_lhs (stmt);
2034 : /* Ignore #pragma omp declare simd functions
2035 : if they don't have data references in the
2036 : call stmt itself. */
2037 261 : if (j == n
2038 131 : && !(op
2039 120 : && (DECL_P (op)
2040 120 : || (REFERENCE_CLASS_P (op)
2041 0 : && get_base_address (op)))))
2042 130 : continue;
2043 : }
2044 : }
2045 : }
2046 63785 : return res;
2047 : }
2048 : /* If dependence analysis will give up due to the limit on the
2049 : number of datarefs stop here and fail fatally. */
2050 4390148 : if (datarefs->length ()
2051 1885471 : > (unsigned)param_loop_max_datarefs_for_datadeps)
2052 0 : return opt_result::failure_at (stmt, "exceeded param "
2053 : "loop-max-datarefs-for-datadeps\n");
2054 : }
2055 222664 : return opt_result::success ();
2056 : }
2057 :
2058 : /* Determine if operating on full vectors for LOOP_VINFO might leave
2059 : some scalar iterations still to do. If so, decide how we should
2060 : handle those scalar iterations. The possibilities are:
2061 :
2062 : (1) Make LOOP_VINFO operate on partial vectors instead of full vectors.
2063 : In this case:
2064 :
2065 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == true
2066 : LOOP_VINFO_PEELING_FOR_NITER == false
2067 :
2068 : (2) Make LOOP_VINFO operate on full vectors and use an epilogue loop
2069 : to handle the remaining scalar iterations. In this case:
2070 :
2071 : LOOP_VINFO_USING_PARTIAL_VECTORS_P == false
2072 : LOOP_VINFO_PEELING_FOR_NITER == true
2073 :
2074 : The MASKED_P argument specifies to what extent
2075 : param_vect_partial_vector_usage is to be honored. For MASKED_P == 0
2076 : no partial vectors are to be used, for MASKED_P == -1 it's
2077 : param_vect_partial_vector_usage that gets to decide whether we may
2078 : consider partial vector usage. For MASKED_P == 1 partial vectors
2079 : may be used if possible.
2080 :
2081 : */
2082 :
2083 : static opt_result
2084 157772 : vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
2085 : int masked_p)
2086 : {
2087 : /* Determine whether there would be any scalar iterations left over. */
2088 157772 : bool need_peeling_or_partial_vectors_p
2089 157772 : = vect_need_peeling_or_partial_vectors_p (loop_vinfo);
2090 :
2091 : /* Decide whether to vectorize the loop with partial vectors. */
2092 157772 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2093 157772 : if (masked_p == 0
2094 157772 : || (masked_p == -1 && param_vect_partial_vector_usage == 0))
2095 : /* If requested explicitly do not use partial vectors. */
2096 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2097 211 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2098 69 : && LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo))
2099 0 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2100 211 : else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2101 69 : && need_peeling_or_partial_vectors_p)
2102 : {
2103 : /* For partial-vector-usage=1, try to push the handling of partial
2104 : vectors to the epilogue, with the main loop continuing to operate
2105 : on full vectors.
2106 :
2107 : If we are unrolling we also do not want to use partial vectors. This
2108 : is to avoid the overhead of generating multiple masks and also to
2109 : avoid having to execute entire iterations of FALSE masked instructions
2110 : when dealing with one or less full iterations.
2111 :
2112 : ??? We could then end up failing to use partial vectors if we
2113 : decide to peel iterations into a prologue, and if the main loop
2114 : then ends up processing fewer than VF iterations. */
2115 47 : if ((param_vect_partial_vector_usage == 1
2116 14 : || loop_vinfo->suggested_unroll_factor > 1)
2117 33 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2118 69 : && !vect_known_niters_smaller_than_vf (loop_vinfo))
2119 : ;
2120 : else
2121 35 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = true;
2122 : }
2123 :
2124 157772 : if (LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo)
2125 0 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2126 0 : return opt_result::failure_at (vect_location,
2127 : "not vectorized: loop needs but cannot "
2128 : "use partial vectors\n");
2129 :
2130 157772 : if (dump_enabled_p ())
2131 12966 : dump_printf_loc (MSG_NOTE, vect_location,
2132 : "operating on %s vectors%s.\n",
2133 12966 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2134 : ? "partial" : "full",
2135 12966 : LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2136 : ? " for epilogue loop" : "");
2137 :
2138 157772 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2139 315544 : = (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2140 157772 : && need_peeling_or_partial_vectors_p);
2141 :
2142 157772 : return opt_result::success ();
2143 : }
2144 :
2145 : /* Function vect_analyze_loop_2.
2146 :
2147 : Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
2148 : analyses will record information in some members of LOOP_VINFO. FATAL
2149 : indicates if some analysis meets fatal error. If one non-NULL pointer
2150 : SUGGESTED_UNROLL_FACTOR is provided, it's intent to be filled with one
2151 : worked out suggested unroll factor, while one NULL pointer shows it's
2152 : going to apply the suggested unroll factor.
2153 : SINGLE_LANE_SLP_DONE_FOR_SUGGESTED_UF is to hold whether single-lane
2154 : slp was forced when the suggested unroll factor was worked out. */
2155 : static opt_result
2156 597484 : vect_analyze_loop_2 (loop_vec_info loop_vinfo, int masked_p, bool &fatal,
2157 : unsigned *suggested_unroll_factor,
2158 : bool& single_lane_slp_done_for_suggested_uf)
2159 : {
2160 597484 : opt_result ok = opt_result::success ();
2161 597484 : int res;
2162 597484 : unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
2163 597484 : loop_vec_info orig_loop_vinfo = NULL;
2164 :
2165 : /* If we are dealing with an epilogue then orig_loop_vinfo points to the
2166 : loop_vec_info of the first vectorized loop. */
2167 597484 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2168 14229 : orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
2169 : else
2170 : orig_loop_vinfo = loop_vinfo;
2171 14229 : gcc_assert (orig_loop_vinfo);
2172 :
2173 : /* We can't mask on niters for uncounted loops due to unknown upper bound. */
2174 597484 : if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
2175 88975 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2176 :
2177 : /* The first group of checks is independent of the vector size. */
2178 597484 : fatal = true;
2179 :
2180 597484 : if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)
2181 597484 : && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo)))
2182 5 : return opt_result::failure_at (vect_location,
2183 : "not vectorized: simd if(0)\n");
2184 :
2185 : /* Find all data references in the loop (which correspond to vdefs/vuses)
2186 : and analyze their evolution in the loop. */
2187 :
2188 597479 : loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
2189 :
2190 : /* Gather the data references. */
2191 597479 : if (!LOOP_VINFO_DATAREFS (loop_vinfo).exists ())
2192 : {
2193 286449 : opt_result res
2194 286449 : = vect_get_datarefs_in_loop (loop, LOOP_VINFO_BBS (loop_vinfo),
2195 : &LOOP_VINFO_DATAREFS (loop_vinfo));
2196 286449 : if (!res)
2197 : {
2198 63785 : if (dump_enabled_p ())
2199 1636 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2200 : "not vectorized: loop contains function "
2201 : "calls or data references that cannot "
2202 : "be analyzed\n");
2203 63785 : return res;
2204 : }
2205 222664 : loop_vinfo->shared->save_datarefs ();
2206 : }
2207 : else
2208 311030 : loop_vinfo->shared->check_datarefs ();
2209 :
2210 : /* Analyze the data references and also adjust the minimal
2211 : vectorization factor according to the loads and stores. */
2212 :
2213 533694 : ok = vect_analyze_data_refs (loop_vinfo, &fatal);
2214 533694 : if (!ok)
2215 : {
2216 75282 : if (dump_enabled_p ())
2217 1328 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2218 : "bad data references.\n");
2219 75282 : return ok;
2220 : }
2221 :
2222 : /* Check if we are applying unroll factor now. */
2223 458412 : bool applying_suggested_uf = loop_vinfo->suggested_unroll_factor > 1;
2224 458412 : gcc_assert (!applying_suggested_uf || !suggested_unroll_factor);
2225 :
2226 : /* When single-lane SLP was forced and we are applying suggested unroll
2227 : factor, keep that decision here. */
2228 916824 : bool force_single_lane = (applying_suggested_uf
2229 458412 : && single_lane_slp_done_for_suggested_uf);
2230 :
2231 : /* Classify all cross-iteration scalar data-flow cycles.
2232 : Cross-iteration cycles caused by virtual phis are analyzed separately. */
2233 458412 : vect_analyze_scalar_cycles (loop_vinfo);
2234 :
2235 458412 : vect_pattern_recog (loop_vinfo);
2236 :
2237 : /* Analyze the access patterns of the data-refs in the loop (consecutive,
2238 : complex, etc.). FORNOW: Only handle consecutive access pattern. */
2239 :
2240 458412 : ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
2241 458412 : if (!ok)
2242 : {
2243 8069 : if (dump_enabled_p ())
2244 292 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2245 : "bad data access.\n");
2246 8069 : return ok;
2247 : }
2248 :
2249 : /* Data-flow analysis to detect stmts that do not need to be vectorized. */
2250 :
2251 450343 : ok = vect_mark_stmts_to_be_vectorized (loop_vinfo, &fatal);
2252 450343 : if (!ok)
2253 : {
2254 47296 : if (dump_enabled_p ())
2255 403 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2256 : "unexpected pattern.\n");
2257 47296 : return ok;
2258 : }
2259 :
2260 : /* While the rest of the analysis below depends on it in some way. */
2261 403047 : fatal = false;
2262 :
2263 : /* Analyze data dependences between the data-refs in the loop
2264 : and adjust the maximum vectorization factor according to
2265 : the dependences.
2266 : FORNOW: fail at the first data dependence that we encounter. */
2267 :
2268 403047 : ok = vect_analyze_data_ref_dependences (loop_vinfo, &max_vf);
2269 403047 : if (!ok)
2270 : {
2271 25300 : if (dump_enabled_p ())
2272 563 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2273 : "bad data dependence.\n");
2274 25300 : return ok;
2275 : }
2276 377747 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
2277 :
2278 : /* Compute the scalar iteration cost. */
2279 377747 : vect_compute_single_scalar_iteration_cost (loop_vinfo);
2280 :
2281 377747 : bool saved_can_use_partial_vectors_p
2282 : = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
2283 :
2284 : /* This is the point where we can re-start analysis with single-lane
2285 : SLP forced. */
2286 524223 : start_over:
2287 :
2288 : /* Check the SLP opportunities in the loop, analyze and build
2289 : SLP trees. */
2290 1048446 : ok = vect_analyze_slp (loop_vinfo, loop_vinfo->stmt_vec_infos.length (),
2291 : force_single_lane);
2292 524223 : if (!ok)
2293 18439 : return ok;
2294 :
2295 : /* If there are any SLP instances mark them as pure_slp and compute
2296 : the overall vectorization factor. */
2297 505784 : if (!vect_make_slp_decision (loop_vinfo))
2298 67402 : return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
2299 :
2300 438382 : if (dump_enabled_p ())
2301 20403 : dump_printf_loc (MSG_NOTE, vect_location, "Loop contains only SLP stmts\n");
2302 :
2303 : /* Dump the vectorization factor from the SLP decision. */
2304 438382 : if (dump_enabled_p ())
2305 : {
2306 20403 : dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
2307 20403 : dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2308 20403 : dump_printf (MSG_NOTE, "\n");
2309 : }
2310 :
2311 : /* We don't expect to have to roll back to anything other than an empty
2312 : set of rgroups. */
2313 438382 : gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
2314 :
2315 : /* Apply the suggested unrolling factor, this was determined by the backend
2316 : during finish_cost the first time we ran the analysis for this
2317 : vector mode. */
2318 438382 : if (applying_suggested_uf)
2319 459 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) *= loop_vinfo->suggested_unroll_factor;
2320 :
2321 : /* Now the vectorization factor is final. */
2322 438382 : poly_uint64 vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2323 438382 : gcc_assert (known_ne (vectorization_factor, 0U));
2324 :
2325 : /* Optimize the SLP graph with the vectorization factor fixed. */
2326 438382 : vect_optimize_slp (loop_vinfo);
2327 :
2328 : /* Gather the loads reachable from the SLP graph entries. */
2329 438382 : vect_gather_slp_loads (loop_vinfo);
2330 :
2331 438382 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && dump_enabled_p ())
2332 : {
2333 14957 : dump_printf_loc (MSG_NOTE, vect_location,
2334 : "vectorization_factor = ");
2335 14957 : dump_dec (MSG_NOTE, vectorization_factor);
2336 14957 : dump_printf (MSG_NOTE, ", niters = %wd\n",
2337 14957 : LOOP_VINFO_INT_NITERS (loop_vinfo));
2338 : }
2339 :
2340 438382 : if (max_vf != MAX_VECTORIZATION_FACTOR
2341 438382 : && maybe_lt (max_vf, LOOP_VINFO_VECT_FACTOR (loop_vinfo)))
2342 41 : return opt_result::failure_at (vect_location, "bad data dependence.\n");
2343 :
2344 438341 : loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
2345 :
2346 : /* Analyze the alignment of the data-refs in the loop. */
2347 438341 : vect_analyze_data_refs_alignment (loop_vinfo);
2348 :
2349 : /* Prune the list of ddrs to be tested at run-time by versioning for alias.
2350 : It is important to call pruning after vect_analyze_data_ref_accesses,
2351 : since we use grouping information gathered by interleaving analysis. */
2352 438341 : ok = vect_prune_runtime_alias_test_list (loop_vinfo);
2353 438341 : if (!ok)
2354 17975 : return ok;
2355 :
2356 : /* Do not invoke vect_enhance_data_refs_alignment for epilogue
2357 : vectorization, since we do not want to add extra peeling or
2358 : add versioning for alignment. */
2359 420366 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
2360 : /* This pass will decide on using loop versioning and/or loop peeling in
2361 : order to enhance the alignment of data references in the loop. */
2362 405173 : ok = vect_enhance_data_refs_alignment (loop_vinfo);
2363 420366 : if (!ok)
2364 0 : return ok;
2365 :
2366 : /* Analyze operations in the SLP instances. We can't simply
2367 : remove unsupported SLP instances as this makes the above
2368 : SLP kind detection invalid and might also affect the VF. */
2369 420366 : if (! vect_slp_analyze_operations (loop_vinfo))
2370 : {
2371 262594 : ok = opt_result::failure_at (vect_location,
2372 : "unsupported SLP instances\n");
2373 262594 : goto again;
2374 : }
2375 :
2376 : /* For now, we don't expect to mix both masking and length approaches for one
2377 : loop, disable it if both are recorded. */
2378 157772 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2379 23577 : && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ()
2380 181343 : && !LOOP_VINFO_LENS (loop_vinfo).is_empty ())
2381 : {
2382 0 : if (dump_enabled_p ())
2383 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2384 : "can't vectorize a loop with partial vectors"
2385 : " because we don't expect to mix different"
2386 : " approaches with partial vectors for the"
2387 : " same loop.\n");
2388 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2389 : }
2390 :
2391 : /* If we still have the option of using partial vectors,
2392 : check whether we can generate the necessary loop controls. */
2393 157772 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
2394 : {
2395 23577 : if (!LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
2396 : {
2397 23571 : if (!vect_verify_full_masking (loop_vinfo)
2398 23571 : && !vect_verify_full_masking_avx512 (loop_vinfo))
2399 6114 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2400 : }
2401 : else /* !LOOP_VINFO_LENS (loop_vinfo).is_empty () */
2402 6 : if (!vect_verify_loop_lens (loop_vinfo))
2403 6 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2404 : }
2405 :
2406 : /* Decide whether this loop_vinfo should use partial vectors or peeling,
2407 : assuming that the loop will be used as a main loop. We will redo
2408 : this analysis later if we instead decide to use the loop as an
2409 : epilogue loop. */
2410 157772 : ok = vect_determine_partial_vectors_and_peeling (loop_vinfo, masked_p);
2411 157772 : if (!ok)
2412 0 : return ok;
2413 :
2414 : /* If we're vectorizing a loop that uses length "controls" and
2415 : can iterate more than once, we apply decrementing IV approach
2416 : in loop control. */
2417 157772 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
2418 35 : && LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_len
2419 0 : && LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) == 0
2420 157772 : && !(LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2421 0 : && known_le (LOOP_VINFO_INT_NITERS (loop_vinfo),
2422 : LOOP_VINFO_VECT_FACTOR (loop_vinfo))))
2423 0 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = true;
2424 :
2425 : /* If a loop uses length controls and has a decrementing loop control IV,
2426 : we will normally pass that IV through a MIN_EXPR to calcaluate the
2427 : basis for the length controls. E.g. in a loop that processes one
2428 : element per scalar iteration, the number of elements would be
2429 : MIN_EXPR <N, VF>, where N is the number of scalar iterations left.
2430 :
2431 : This MIN_EXPR approach allows us to use pointer IVs with an invariant
2432 : step, since only the final iteration of the vector loop can have
2433 : inactive lanes.
2434 :
2435 : However, some targets have a dedicated instruction for calculating the
2436 : preferred length, given the total number of elements that still need to
2437 : be processed. This is encapsulated in the SELECT_VL internal function.
2438 :
2439 : If the target supports SELECT_VL, we can use it instead of MIN_EXPR
2440 : to determine the basis for the length controls. However, unlike the
2441 : MIN_EXPR calculation, the SELECT_VL calculation can decide to make
2442 : lanes inactive in any iteration of the vector loop, not just the last
2443 : iteration. This SELECT_VL approach therefore requires us to use pointer
2444 : IVs with variable steps.
2445 :
2446 : Once we've decided how many elements should be processed by one
2447 : iteration of the vector loop, we need to populate the rgroup controls.
2448 : If a loop has multiple rgroups, we need to make sure that those rgroups
2449 : "line up" (that is, they must be consistent about which elements are
2450 : active and which aren't). This is done by vect_adjust_loop_lens_control.
2451 :
2452 : In principle, it would be possible to use vect_adjust_loop_lens_control
2453 : on either the result of a MIN_EXPR or the result of a SELECT_VL.
2454 : However:
2455 :
2456 : (1) In practice, it only makes sense to use SELECT_VL when a vector
2457 : operation will be controlled directly by the result. It is not
2458 : worth using SELECT_VL if it would only be the input to other
2459 : calculations.
2460 :
2461 : (2) If we use SELECT_VL for an rgroup that has N controls, each associated
2462 : pointer IV will need N updates by a variable amount (N-1 updates
2463 : within the iteration and 1 update to move to the next iteration).
2464 :
2465 : Because of this, we prefer to use the MIN_EXPR approach whenever there
2466 : is more than one length control.
2467 :
2468 : In addition, SELECT_VL always operates to a granularity of 1 unit.
2469 : If we wanted to use it to control an SLP operation on N consecutive
2470 : elements, we would need to make the SELECT_VL inputs measure scalar
2471 : iterations (rather than elements) and then multiply the SELECT_VL
2472 : result by N. But using SELECT_VL this way is inefficient because
2473 : of (1) above.
2474 :
2475 : 2. We don't apply SELECT_VL on single-rgroup when both (1) and (2) are
2476 : satisfied:
2477 :
2478 : (1). LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) is true.
2479 : (2). LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant () is true.
2480 :
2481 : Since SELECT_VL (variable step) will make SCEV analysis failed and then
2482 : we will fail to gain benefits of following unroll optimizations. We prefer
2483 : using the MIN_EXPR approach in this situation. */
2484 157772 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
2485 : {
2486 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
2487 0 : if (LOOP_VINFO_LENS (loop_vinfo).length () == 1
2488 0 : && LOOP_VINFO_LENS (loop_vinfo)[0].factor == 1
2489 0 : && (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2490 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ()))
2491 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = true;
2492 :
2493 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2494 0 : for (auto rgc : LOOP_VINFO_LENS (loop_vinfo))
2495 0 : if (rgc.type
2496 0 : && !direct_internal_fn_supported_p (IFN_SELECT_VL,
2497 : rgc.type, iv_type,
2498 : OPTIMIZE_FOR_SPEED))
2499 : {
2500 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2501 0 : break;
2502 : }
2503 :
2504 : /* If any of the SLP instances cover more than a single lane
2505 : we cannot use .SELECT_VL at the moment, even if the number
2506 : of lanes is uniform throughout the SLP graph. */
2507 0 : if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
2508 0 : for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
2509 0 : if (SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)) != 1
2510 0 : && !(SLP_INSTANCE_KIND (inst) == slp_inst_kind_store
2511 0 : && SLP_INSTANCE_TREE (inst)->ldst_lanes))
2512 : {
2513 0 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2514 0 : break;
2515 : }
2516 : }
2517 :
2518 : /* If we're vectorizing an epilogue loop, the vectorized loop either needs
2519 : to be able to handle fewer than VF scalars, or needs to have a lower VF
2520 : than the main loop. */
2521 157772 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2522 13789 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2523 : {
2524 13773 : poly_uint64 unscaled_vf
2525 13773 : = exact_div (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo),
2526 : orig_loop_vinfo->suggested_unroll_factor);
2527 13773 : if (maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo), unscaled_vf))
2528 400 : return opt_result::failure_at (vect_location,
2529 : "Vectorization factor too high for"
2530 : " epilogue loop.\n");
2531 : }
2532 :
2533 : /* If the epilogue needs peeling for gaps but the main loop doesn't give
2534 : up on the epilogue. */
2535 157372 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2536 13389 : && LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2537 73 : && (LOOP_VINFO_PEELING_FOR_GAPS (orig_loop_vinfo)
2538 : != LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)))
2539 4 : return opt_result::failure_at (vect_location,
2540 : "Epilogue loop requires peeling for gaps "
2541 : "but main loop does not.\n");
2542 :
2543 : /* If an epilogue loop is required make sure we can create one. */
2544 157368 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
2545 156062 : || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
2546 57429 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2547 : {
2548 101469 : if (dump_enabled_p ())
2549 5882 : dump_printf_loc (MSG_NOTE, vect_location, "epilog loop required\n");
2550 101469 : if (!vect_can_advance_ivs_p (loop_vinfo)
2551 202408 : || !slpeel_can_duplicate_loop_p (loop,
2552 : LOOP_VINFO_MAIN_EXIT (loop_vinfo),
2553 100939 : LOOP_VINFO_MAIN_EXIT (loop_vinfo)))
2554 : {
2555 530 : ok = opt_result::failure_at (vect_location,
2556 : "not vectorized: can't create required "
2557 : "epilog loop\n");
2558 530 : goto again;
2559 : }
2560 : }
2561 :
2562 : /* Check the costings of the loop make vectorizing worthwhile. */
2563 156838 : res = vect_analyze_loop_costing (loop_vinfo, suggested_unroll_factor);
2564 156838 : if (res < 0 && !param_vect_allow_possibly_not_worthwhile_vectorizations)
2565 : {
2566 29118 : ok = opt_result::failure_at (vect_location,
2567 : "Loop costings may not be worthwhile.\n");
2568 29118 : goto again;
2569 : }
2570 127720 : if (!res)
2571 28204 : return opt_result::failure_at (vect_location,
2572 : "Loop costings not worthwhile.\n");
2573 :
2574 : /* During peeling, we need to check if number of loop iterations is
2575 : enough for both peeled prolog loop and vector loop. This check
2576 : can be merged along with threshold check of loop versioning, so
2577 : increase threshold for this case if necessary.
2578 :
2579 : If we are analyzing an epilogue we still want to check what its
2580 : versioning threshold would be. If we decide to vectorize the epilogues we
2581 : will want to use the lowest versioning threshold of all epilogues and main
2582 : loop. This will enable us to enter a vectorized epilogue even when
2583 : versioning the loop. We can't simply check whether the epilogue requires
2584 : versioning though since we may have skipped some versioning checks when
2585 : analyzing the epilogue. For instance, checks for alias versioning will be
2586 : skipped when dealing with epilogues as we assume we already checked them
2587 : for the main loop. So instead we always check the 'orig_loop_vinfo'. */
2588 99516 : if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo))
2589 : {
2590 9099 : poly_uint64 niters_th = 0;
2591 9099 : unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2592 :
2593 9099 : if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
2594 : {
2595 : /* Niters for peeled prolog loop. */
2596 9099 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
2597 : {
2598 115 : dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
2599 115 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
2600 115 : niters_th += TYPE_VECTOR_SUBPARTS (vectype) - 1;
2601 : }
2602 : else
2603 8984 : niters_th += LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
2604 : }
2605 :
2606 : /* Niters for at least one iteration of vectorized loop. */
2607 9099 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
2608 9095 : niters_th += LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2609 : /* One additional iteration because of peeling for gap. */
2610 9099 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
2611 75 : niters_th += 1;
2612 :
2613 : /* Use the same condition as vect_transform_loop to decide when to use
2614 : the cost to determine a versioning threshold. */
2615 9099 : if (vect_apply_runtime_profitability_check_p (loop_vinfo)
2616 9099 : && ordered_p (th, niters_th))
2617 6772 : niters_th = ordered_max (poly_uint64 (th), niters_th);
2618 :
2619 9099 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = niters_th;
2620 : }
2621 :
2622 99516 : gcc_assert (known_eq (vectorization_factor,
2623 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
2624 :
2625 99516 : single_lane_slp_done_for_suggested_uf = force_single_lane;
2626 :
2627 : /* Ok to vectorize! */
2628 99516 : LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
2629 99516 : return opt_result::success ();
2630 :
2631 292242 : again:
2632 : /* Ensure that "ok" is false (with an opt_problem if dumping is enabled). */
2633 292242 : gcc_assert (!ok);
2634 :
2635 : /* Try again with single-lane SLP. */
2636 292242 : if (force_single_lane)
2637 144790 : return ok;
2638 :
2639 : /* If we are applying suggested unroll factor, we don't need to
2640 : re-try any more as we want to keep the SLP mode fixed. */
2641 147452 : if (applying_suggested_uf)
2642 10 : return ok;
2643 :
2644 : /* Likewise if the grouped loads or stores in the SLP cannot be handled
2645 : via interleaving or lane instructions. */
2646 : slp_instance instance;
2647 : slp_tree node;
2648 : unsigned i, j;
2649 397352 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
2650 : {
2651 250876 : if (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance)) != vect_internal_def)
2652 0 : continue;
2653 :
2654 250876 : stmt_vec_info vinfo;
2655 250876 : vinfo = SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance))[0];
2656 250876 : if (!vinfo || !STMT_VINFO_GROUPED_ACCESS (vinfo))
2657 248088 : continue;
2658 2788 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2659 2788 : unsigned int size = DR_GROUP_SIZE (vinfo);
2660 2788 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
2661 2788 : if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
2662 4898 : && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
2663 5570 : && ! vect_grouped_store_supported (vectype, size))
2664 672 : return opt_result::failure_at (vinfo->stmt,
2665 : "unsupported grouped store\n");
2666 253503 : FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, node)
2667 : {
2668 2308 : vinfo = SLP_TREE_REPRESENTATIVE (node);
2669 2308 : if (STMT_VINFO_GROUPED_ACCESS (vinfo))
2670 : {
2671 2004 : vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
2672 2004 : bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
2673 2004 : size = DR_GROUP_SIZE (vinfo);
2674 2004 : vectype = SLP_TREE_VECTYPE (node);
2675 2004 : if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
2676 2004 : && ! vect_grouped_load_supported (vectype, single_element_p,
2677 : size))
2678 294 : return opt_result::failure_at (vinfo->stmt,
2679 : "unsupported grouped load\n");
2680 : }
2681 : }
2682 : }
2683 :
2684 : /* Roll back state appropriately. Force single-lane SLP this time. */
2685 146476 : force_single_lane = true;
2686 146476 : if (dump_enabled_p ())
2687 3817 : dump_printf_loc (MSG_NOTE, vect_location,
2688 : "re-trying with single-lane SLP\n");
2689 :
2690 : /* Reset the vectorization factor. */
2691 146476 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
2692 : /* Free the SLP instances. */
2693 396379 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
2694 249903 : vect_free_slp_instance (instance);
2695 146476 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
2696 : /* Reset altered state on stmts. */
2697 702080 : for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
2698 : {
2699 409128 : basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
2700 409128 : for (gimple_stmt_iterator si = gsi_start_phis (bb);
2701 738140 : !gsi_end_p (si); gsi_next (&si))
2702 : {
2703 329012 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
2704 329012 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
2705 329012 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
2706 : {
2707 : /* vectorizable_reduction adjusts reduction stmt def-types,
2708 : restore them to that of the PHI. */
2709 26787 : STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info))
2710 26787 : = STMT_VINFO_DEF_TYPE (stmt_info);
2711 26787 : STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
2712 : (STMT_VINFO_REDUC_DEF (stmt_info)))
2713 26787 : = STMT_VINFO_DEF_TYPE (stmt_info);
2714 : }
2715 : }
2716 : }
2717 : /* Free optimized alias test DDRS. */
2718 146476 : LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
2719 146476 : LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
2720 146476 : LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).release ();
2721 : /* Reset target cost data. */
2722 146476 : delete loop_vinfo->vector_costs;
2723 146476 : loop_vinfo->vector_costs = nullptr;
2724 : /* Reset accumulated rgroup information. */
2725 146476 : LOOP_VINFO_MASKS (loop_vinfo).mask_set.empty ();
2726 146476 : release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo).rgc_vec);
2727 146476 : release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo));
2728 : /* Reset assorted flags. */
2729 146476 : LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo) = false;
2730 146476 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = false;
2731 146476 : LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo) = 0;
2732 146476 : LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo) = 0;
2733 146476 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2734 146476 : = saved_can_use_partial_vectors_p;
2735 146476 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2736 146476 : LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
2737 146476 : LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
2738 146476 : LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
2739 :
2740 146476 : if (loop_vinfo->scan_map)
2741 122 : loop_vinfo->scan_map->empty ();
2742 :
2743 146476 : goto start_over;
2744 : }
2745 :
2746 : /* Return true if vectorizing a loop using NEW_LOOP_VINFO appears
2747 : to be better than vectorizing it using OLD_LOOP_VINFO. Assume that
2748 : OLD_LOOP_VINFO is better unless something specifically indicates
2749 : otherwise.
2750 :
2751 : Note that this deliberately isn't a partial order. */
2752 :
2753 : static bool
2754 35594 : vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo,
2755 : loop_vec_info old_loop_vinfo)
2756 : {
2757 35594 : struct loop *loop = LOOP_VINFO_LOOP (new_loop_vinfo);
2758 35594 : gcc_assert (LOOP_VINFO_LOOP (old_loop_vinfo) == loop);
2759 :
2760 35594 : poly_int64 new_vf = LOOP_VINFO_VECT_FACTOR (new_loop_vinfo);
2761 35594 : poly_int64 old_vf = LOOP_VINFO_VECT_FACTOR (old_loop_vinfo);
2762 :
2763 : /* Always prefer a VF of loop->simdlen over any other VF. */
2764 35594 : if (loop->simdlen)
2765 : {
2766 0 : bool new_simdlen_p = known_eq (new_vf, loop->simdlen);
2767 0 : bool old_simdlen_p = known_eq (old_vf, loop->simdlen);
2768 0 : if (new_simdlen_p != old_simdlen_p)
2769 : return new_simdlen_p;
2770 : }
2771 :
2772 35594 : const auto *old_costs = old_loop_vinfo->vector_costs;
2773 35594 : const auto *new_costs = new_loop_vinfo->vector_costs;
2774 35594 : if (loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo))
2775 1547 : return new_costs->better_epilogue_loop_than_p (old_costs, main_loop);
2776 :
2777 34047 : return new_costs->better_main_loop_than_p (old_costs);
2778 : }
2779 :
2780 : /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO. Return
2781 : true if we should. */
2782 :
2783 : static bool
2784 35594 : vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
2785 : loop_vec_info old_loop_vinfo)
2786 : {
2787 35594 : if (!vect_better_loop_vinfo_p (new_loop_vinfo, old_loop_vinfo))
2788 : return false;
2789 :
2790 1444 : if (dump_enabled_p ())
2791 18 : dump_printf_loc (MSG_NOTE, vect_location,
2792 : "***** Preferring vector mode %s to vector mode %s\n",
2793 18 : GET_MODE_NAME (new_loop_vinfo->vector_mode),
2794 18 : GET_MODE_NAME (old_loop_vinfo->vector_mode));
2795 : return true;
2796 : }
2797 :
2798 : /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
2799 : not NULL. When MASKED_P is not -1 override the default
2800 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
2801 : Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
2802 : mode useful to analyze.
2803 : Return the loop_vinfo on success and wrapped null on failure. */
2804 :
2805 : static opt_loop_vec_info
2806 597025 : vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
2807 : const vect_loop_form_info *loop_form_info,
2808 : loop_vec_info orig_loop_vinfo,
2809 : const vector_modes &vector_modes, unsigned &mode_i,
2810 : int masked_p,
2811 : machine_mode &autodetected_vector_mode,
2812 : bool &fatal)
2813 : {
2814 597025 : loop_vec_info loop_vinfo
2815 597025 : = vect_create_loop_vinfo (loop, shared, loop_form_info, orig_loop_vinfo);
2816 :
2817 597025 : machine_mode vector_mode = vector_modes[mode_i];
2818 597025 : loop_vinfo->vector_mode = vector_mode;
2819 597025 : unsigned int suggested_unroll_factor = 1;
2820 597025 : bool single_lane_slp_done_for_suggested_uf = false;
2821 :
2822 : /* Run the main analysis. */
2823 597025 : opt_result res = vect_analyze_loop_2 (loop_vinfo, masked_p, fatal,
2824 : &suggested_unroll_factor,
2825 : single_lane_slp_done_for_suggested_uf);
2826 597025 : if (dump_enabled_p ())
2827 22131 : dump_printf_loc (MSG_NOTE, vect_location,
2828 : "***** Analysis %s with vector mode %s\n",
2829 22131 : res ? "succeeded" : "failed",
2830 22131 : GET_MODE_NAME (loop_vinfo->vector_mode));
2831 :
2832 597025 : auto user_unroll = LOOP_VINFO_LOOP (loop_vinfo)->unroll;
2833 597025 : if (res && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
2834 : /* Check to see if the user wants to unroll or if the target wants to. */
2835 687732 : && (suggested_unroll_factor > 1 || user_unroll > 1))
2836 : {
2837 485 : if (suggested_unroll_factor == 1)
2838 : {
2839 66 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
2840 66 : suggested_unroll_factor = user_unroll / assumed_vf;
2841 66 : if (suggested_unroll_factor > 1)
2842 : {
2843 40 : if (dump_enabled_p ())
2844 20 : dump_printf_loc (MSG_NOTE, vect_location,
2845 : "setting unroll factor to %d based on user requested "
2846 : "unroll factor %d and suggested vectorization "
2847 : "factor: %d\n",
2848 : suggested_unroll_factor, user_unroll, assumed_vf);
2849 : }
2850 : }
2851 :
2852 485 : if (suggested_unroll_factor > 1)
2853 : {
2854 459 : if (dump_enabled_p ())
2855 62 : dump_printf_loc (MSG_NOTE, vect_location,
2856 : "***** Re-trying analysis for unrolling"
2857 : " with unroll factor %d and %s slp.\n",
2858 : suggested_unroll_factor,
2859 : single_lane_slp_done_for_suggested_uf
2860 : ? "single-lane" : "");
2861 459 : loop_vec_info unroll_vinfo
2862 459 : = vect_create_loop_vinfo (loop, shared, loop_form_info, NULL);
2863 459 : unroll_vinfo->vector_mode = vector_mode;
2864 459 : unroll_vinfo->suggested_unroll_factor = suggested_unroll_factor;
2865 459 : opt_result new_res
2866 459 : = vect_analyze_loop_2 (unroll_vinfo, masked_p, fatal, NULL,
2867 : single_lane_slp_done_for_suggested_uf);
2868 459 : if (new_res)
2869 : {
2870 400 : delete loop_vinfo;
2871 : loop_vinfo = unroll_vinfo;
2872 : }
2873 : else
2874 59 : delete unroll_vinfo;
2875 : }
2876 :
2877 : /* Record that we have honored a user unroll factor. */
2878 485 : LOOP_VINFO_USER_UNROLL (loop_vinfo) = user_unroll > 1;
2879 : }
2880 :
2881 : /* Remember the autodetected vector mode. */
2882 597025 : if (vector_mode == VOIDmode)
2883 276616 : autodetected_vector_mode = loop_vinfo->vector_mode;
2884 :
2885 : /* Advance mode_i, first skipping modes that would result in the
2886 : same analysis result. */
2887 2632055 : while (mode_i + 1 < vector_modes.length ()
2888 1810254 : && vect_chooses_same_modes_p (loop_vinfo,
2889 792739 : vector_modes[mode_i + 1]))
2890 : {
2891 420490 : if (dump_enabled_p ())
2892 17446 : dump_printf_loc (MSG_NOTE, vect_location,
2893 : "***** The result for vector mode %s would"
2894 : " be the same\n",
2895 17446 : GET_MODE_NAME (vector_modes[mode_i + 1]));
2896 420490 : mode_i += 1;
2897 : }
2898 597025 : if (mode_i + 1 < vector_modes.length ()
2899 969274 : && vect_chooses_same_modes_p (autodetected_vector_mode,
2900 372249 : vector_modes[mode_i + 1]))
2901 : {
2902 442 : if (dump_enabled_p ())
2903 11 : dump_printf_loc (MSG_NOTE, vect_location,
2904 : "***** Skipping vector mode %s, which would"
2905 : " repeat the analysis for %s\n",
2906 11 : GET_MODE_NAME (vector_modes[mode_i + 1]),
2907 11 : GET_MODE_NAME (autodetected_vector_mode));
2908 442 : mode_i += 1;
2909 : }
2910 597025 : mode_i++;
2911 :
2912 597025 : if (!res)
2913 : {
2914 497909 : delete loop_vinfo;
2915 497909 : if (fatal)
2916 105975 : gcc_checking_assert (orig_loop_vinfo == NULL);
2917 497909 : return opt_loop_vec_info::propagate_failure (res);
2918 : }
2919 :
2920 99116 : return opt_loop_vec_info::success (loop_vinfo);
2921 : }
2922 :
2923 : /* Function vect_analyze_loop.
2924 :
2925 : Apply a set of analyses on LOOP, and create a loop_vec_info struct
2926 : for it. The different analyses will record information in the
2927 : loop_vec_info struct. */
2928 : opt_loop_vec_info
2929 479981 : vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
2930 : vec_info_shared *shared)
2931 : {
2932 479981 : DUMP_VECT_SCOPE ("analyze_loop_nest");
2933 :
2934 479981 : if (loop_outer (loop)
2935 479981 : && loop_vec_info_for_loop (loop_outer (loop))
2936 480565 : && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
2937 584 : return opt_loop_vec_info::failure_at (vect_location,
2938 : "outer-loop already vectorized.\n");
2939 :
2940 479397 : if (!find_loop_nest (loop, &shared->loop_nest))
2941 21501 : return opt_loop_vec_info::failure_at
2942 21501 : (vect_location,
2943 : "not vectorized: loop nest containing two or more consecutive inner"
2944 : " loops cannot be vectorized\n");
2945 :
2946 : /* Analyze the loop form. */
2947 457896 : vect_loop_form_info loop_form_info;
2948 457896 : opt_result res = vect_analyze_loop_form (loop, loop_vectorized_call,
2949 : &loop_form_info);
2950 457896 : if (!res)
2951 : {
2952 181280 : if (dump_enabled_p ())
2953 1546 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2954 : "bad loop form.\n");
2955 181280 : return opt_loop_vec_info::propagate_failure (res);
2956 : }
2957 276616 : if (!integer_onep (loop_form_info.assumptions))
2958 : {
2959 : /* We consider to vectorize this loop by versioning it under
2960 : some assumptions. In order to do this, we need to clear
2961 : existing information computed by scev and niter analyzer. */
2962 8908 : scev_reset_htab ();
2963 8908 : free_numbers_of_iterations_estimates (loop);
2964 : /* Also set flag for this loop so that following scev and niter
2965 : analysis are done under the assumptions. */
2966 8908 : loop_constraint_set (loop, LOOP_C_FINITE);
2967 : }
2968 : else
2969 : /* Clear the existing niter information to make sure the nonwrapping flag
2970 : will be calculated and set propriately. */
2971 267708 : free_numbers_of_iterations_estimates (loop);
2972 :
2973 276616 : auto_vector_modes vector_modes;
2974 : /* Autodetect first vector size we try. */
2975 276616 : vector_modes.safe_push (VOIDmode);
2976 276616 : unsigned int autovec_flags
2977 553232 : = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
2978 276616 : loop->simdlen != 0);
2979 276616 : bool pick_lowest_cost_p = ((autovec_flags & VECT_COMPARE_COSTS)
2980 276616 : && !unlimited_cost_model (loop));
2981 276616 : machine_mode autodetected_vector_mode = VOIDmode;
2982 276616 : opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
2983 276616 : unsigned int mode_i = 0;
2984 276616 : unsigned HOST_WIDE_INT simdlen = loop->simdlen;
2985 :
2986 : /* Keep track of the VF for each mode. Initialize all to 0 which indicates
2987 : a mode has not been analyzed. */
2988 276616 : auto_vec<poly_uint64, 8> cached_vf_per_mode;
2989 2777814 : for (unsigned i = 0; i < vector_modes.length (); ++i)
2990 1112291 : cached_vf_per_mode.safe_push (0);
2991 :
2992 : /* First determine the main loop vectorization mode, either the first
2993 : one that works, starting with auto-detecting the vector mode and then
2994 : following the targets order of preference, or the one with the
2995 : lowest cost if pick_lowest_cost_p. */
2996 888976 : while (1)
2997 : {
2998 582796 : bool fatal;
2999 582796 : unsigned int last_mode_i = mode_i;
3000 : /* Set cached VF to -1 prior to analysis, which indicates a mode has
3001 : failed. */
3002 582796 : cached_vf_per_mode[last_mode_i] = -1;
3003 582796 : opt_loop_vec_info loop_vinfo
3004 582796 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3005 : NULL, vector_modes, mode_i, -1,
3006 : autodetected_vector_mode, fatal);
3007 582796 : if (fatal)
3008 : break;
3009 :
3010 476821 : if (loop_vinfo)
3011 : {
3012 : /* Analysis has been successful so update the VF value. The
3013 : VF should always be a multiple of unroll_factor and we want to
3014 : capture the original VF here. */
3015 90707 : cached_vf_per_mode[last_mode_i]
3016 90707 : = exact_div (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
3017 90707 : loop_vinfo->suggested_unroll_factor);
3018 : /* Once we hit the desired simdlen for the first time,
3019 : discard any previous attempts. */
3020 90707 : if (simdlen
3021 90707 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), simdlen))
3022 : {
3023 47 : delete first_loop_vinfo;
3024 90707 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3025 90707 : simdlen = 0;
3026 : }
3027 90660 : else if (pick_lowest_cost_p
3028 76469 : && first_loop_vinfo
3029 124707 : && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
3030 : {
3031 : /* Pick loop_vinfo over first_loop_vinfo. */
3032 1274 : delete first_loop_vinfo;
3033 1274 : first_loop_vinfo = opt_loop_vec_info::success (NULL);
3034 : }
3035 90707 : if (first_loop_vinfo == NULL)
3036 : first_loop_vinfo = loop_vinfo;
3037 : else
3038 : {
3039 32775 : delete loop_vinfo;
3040 32775 : loop_vinfo = opt_loop_vec_info::success (NULL);
3041 : }
3042 :
3043 : /* Commit to first_loop_vinfo if we have no reason to try
3044 : alternatives. */
3045 90707 : if (!simdlen && !pick_lowest_cost_p)
3046 : break;
3047 : }
3048 462592 : if (mode_i == vector_modes.length ()
3049 462592 : || autodetected_vector_mode == VOIDmode)
3050 : break;
3051 :
3052 : /* Try the next biggest vector size. */
3053 306180 : if (dump_enabled_p ())
3054 5038 : dump_printf_loc (MSG_NOTE, vect_location,
3055 : "***** Re-trying analysis with vector mode %s\n",
3056 5038 : GET_MODE_NAME (vector_modes[mode_i]));
3057 306180 : }
3058 276616 : if (!first_loop_vinfo)
3059 219963 : return opt_loop_vec_info::propagate_failure (res);
3060 :
3061 56653 : if (dump_enabled_p ())
3062 9801 : dump_printf_loc (MSG_NOTE, vect_location,
3063 : "***** Choosing vector mode %s\n",
3064 9801 : GET_MODE_NAME (first_loop_vinfo->vector_mode));
3065 :
3066 : /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
3067 : enabled, SIMDUID is not set, it is the innermost loop and we have
3068 : either already found the loop's SIMDLEN or there was no SIMDLEN to
3069 : begin with.
3070 : TODO: Enable epilogue vectorization for loops with SIMDUID set. */
3071 56653 : bool vect_epilogues = (!simdlen
3072 56651 : && loop->inner == NULL
3073 56065 : && param_vect_epilogues_nomask
3074 54923 : && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
3075 : /* No code motion support for multiple epilogues so for now
3076 : not supported when multiple exits. */
3077 26850 : && !LOOP_VINFO_EARLY_BREAKS (first_loop_vinfo)
3078 26377 : && !loop->simduid
3079 81614 : && loop_cost_model (loop) > VECT_COST_MODEL_VERY_CHEAP);
3080 56653 : if (!vect_epilogues)
3081 43455 : return first_loop_vinfo;
3082 :
3083 : /* Now analyze first_loop_vinfo for epilogue vectorization. */
3084 :
3085 : /* For epilogues start the analysis from the first mode. The motivation
3086 : behind starting from the beginning comes from cases where the VECTOR_MODES
3087 : array may contain length-agnostic and length-specific modes. Their
3088 : ordering is not guaranteed, so we could end up picking a mode for the main
3089 : loop that is after the epilogue's optimal mode. */
3090 13198 : int masked_p = -1;
3091 13198 : if (!unlimited_cost_model (loop)
3092 13198 : && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3093 : != VOIDmode))
3094 : {
3095 5 : vector_modes[0]
3096 5 : = first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3097 5 : cached_vf_per_mode[0] = 0;
3098 : }
3099 : else
3100 13193 : vector_modes[0] = autodetected_vector_mode;
3101 13198 : mode_i = 0;
3102 :
3103 13235 : bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
3104 13198 : || masked_p == 1);
3105 : if (supports_partial_vectors
3106 37 : && !partial_vectors_supported_p ()
3107 37 : && !LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (first_loop_vinfo))
3108 : supports_partial_vectors = false;
3109 13198 : poly_uint64 first_vinfo_vf = LOOP_VINFO_VECT_FACTOR (first_loop_vinfo);
3110 :
3111 13198 : loop_vec_info orig_loop_vinfo = first_loop_vinfo;
3112 13394 : do
3113 : {
3114 : /* Let the user override what the target suggests. */
3115 13296 : if (OPTION_SET_P (param_vect_partial_vector_usage))
3116 48 : masked_p = -1;
3117 :
3118 51400 : while (1)
3119 : {
3120 : /* If the target does not support partial vectors we can shorten the
3121 : number of modes to analyze for the epilogue as we know we can't
3122 : pick a mode that would lead to a VF at least as big as the
3123 : FIRST_VINFO_VF. */
3124 68360 : if (!supports_partial_vectors
3125 51400 : && maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
3126 : {
3127 24170 : mode_i++;
3128 48340 : if (mode_i == vector_modes.length ())
3129 : break;
3130 29961 : continue;
3131 : }
3132 : /* We would need an exhaustive search to find all modes we
3133 : skipped but that would lead to the same result as the
3134 : analysis it was skipped for and where we'd could check
3135 : cached_vf_per_mode against.
3136 : Check for the autodetected mode, which is the common
3137 : situation on x86 which does not perform cost comparison. */
3138 40231 : if (!supports_partial_vectors
3139 27182 : && maybe_ge (cached_vf_per_mode[0], first_vinfo_vf)
3140 53565 : && vect_chooses_same_modes_p (autodetected_vector_mode,
3141 26335 : vector_modes[mode_i]))
3142 : {
3143 13001 : mode_i++;
3144 26002 : if (mode_i == vector_modes.length ())
3145 : break;
3146 13001 : continue;
3147 : }
3148 :
3149 14229 : if (dump_enabled_p ())
3150 3513 : dump_printf_loc (MSG_NOTE, vect_location,
3151 : "***** Re-trying epilogue analysis with vector "
3152 3513 : "mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
3153 :
3154 14229 : bool fatal;
3155 14229 : opt_loop_vec_info loop_vinfo
3156 14229 : = vect_analyze_loop_1 (loop, shared, &loop_form_info,
3157 : orig_loop_vinfo,
3158 : vector_modes, mode_i, masked_p,
3159 : autodetected_vector_mode, fatal);
3160 14229 : if (fatal)
3161 : break;
3162 :
3163 14229 : if (loop_vinfo)
3164 : {
3165 8409 : if (pick_lowest_cost_p
3166 5384 : && orig_loop_vinfo->epilogue_vinfo
3167 9956 : && vect_joust_loop_vinfos (loop_vinfo,
3168 1547 : orig_loop_vinfo->epilogue_vinfo))
3169 : {
3170 170 : gcc_assert (vect_epilogues);
3171 170 : delete orig_loop_vinfo->epilogue_vinfo;
3172 170 : orig_loop_vinfo->epilogue_vinfo = nullptr;
3173 : }
3174 8409 : if (!orig_loop_vinfo->epilogue_vinfo)
3175 7032 : orig_loop_vinfo->epilogue_vinfo = loop_vinfo;
3176 : else
3177 : {
3178 1377 : delete loop_vinfo;
3179 1377 : loop_vinfo = opt_loop_vec_info::success (NULL);
3180 : }
3181 :
3182 : /* For now only allow one epilogue loop, but allow
3183 : pick_lowest_cost_p to replace it, so commit to the
3184 : first epilogue if we have no reason to try alternatives. */
3185 8409 : if (!pick_lowest_cost_p)
3186 : break;
3187 : }
3188 :
3189 : /* Revert back to the default from the suggested preferred
3190 : epilogue vectorization mode. */
3191 11204 : masked_p = -1;
3192 22408 : if (mode_i == vector_modes.length ())
3193 : break;
3194 : }
3195 :
3196 13296 : orig_loop_vinfo = orig_loop_vinfo->epilogue_vinfo;
3197 13296 : if (!orig_loop_vinfo)
3198 : break;
3199 :
3200 : /* When we selected a first vectorized epilogue, see if the target
3201 : suggests to have another one. */
3202 6862 : masked_p = -1;
3203 6862 : if (!unlimited_cost_model (loop)
3204 3843 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
3205 10698 : && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
3206 : != VOIDmode))
3207 : {
3208 196 : vector_modes[0]
3209 98 : = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
3210 98 : cached_vf_per_mode[0] = 0;
3211 98 : mode_i = 0;
3212 : }
3213 : else
3214 : break;
3215 98 : }
3216 : while (1);
3217 :
3218 13198 : if (first_loop_vinfo->epilogue_vinfo)
3219 : {
3220 6771 : poly_uint64 lowest_th
3221 6771 : = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
3222 6771 : loop_vec_info epilog_vinfo = first_loop_vinfo->epilogue_vinfo;
3223 6862 : do
3224 : {
3225 6862 : poly_uint64 th = LOOP_VINFO_VERSIONING_THRESHOLD (epilog_vinfo);
3226 6862 : gcc_assert (!LOOP_REQUIRES_VERSIONING (epilog_vinfo)
3227 : || maybe_ne (lowest_th, 0U));
3228 : /* Keep track of the known smallest versioning threshold. */
3229 6862 : if (ordered_p (lowest_th, th))
3230 6862 : lowest_th = ordered_min (lowest_th, th);
3231 6862 : epilog_vinfo = epilog_vinfo->epilogue_vinfo;
3232 : }
3233 6862 : while (epilog_vinfo);
3234 6771 : LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
3235 6771 : if (dump_enabled_p ())
3236 1531 : dump_printf_loc (MSG_NOTE, vect_location,
3237 : "***** Choosing epilogue vector mode %s\n",
3238 1531 : GET_MODE_NAME
3239 : (first_loop_vinfo->epilogue_vinfo->vector_mode));
3240 : }
3241 :
3242 13198 : return first_loop_vinfo;
3243 734512 : }
3244 :
3245 : /* Return true if there is an in-order reduction function for CODE, storing
3246 : it in *REDUC_FN if so. */
3247 :
3248 : static bool
3249 5192 : fold_left_reduction_fn (code_helper code, internal_fn *reduc_fn)
3250 : {
3251 : /* We support MINUS_EXPR by negating the operand. This also preserves an
3252 : initial -0.0 since -0.0 - 0.0 (neutral op for MINUS_EXPR) == -0.0 +
3253 : (-0.0) = -0.0. */
3254 5192 : if (code == PLUS_EXPR || code == MINUS_EXPR)
3255 : {
3256 4520 : *reduc_fn = IFN_FOLD_LEFT_PLUS;
3257 0 : return true;
3258 : }
3259 : return false;
3260 : }
3261 :
3262 : /* Function reduction_fn_for_scalar_code
3263 :
3264 : Input:
3265 : CODE - tree_code of a reduction operations.
3266 :
3267 : Output:
3268 : REDUC_FN - the corresponding internal function to be used to reduce the
3269 : vector of partial results into a single scalar result, or IFN_LAST
3270 : if the operation is a supported reduction operation, but does not have
3271 : such an internal function.
3272 :
3273 : Return FALSE if CODE currently cannot be vectorized as reduction. */
3274 :
3275 : bool
3276 2283305 : reduction_fn_for_scalar_code (code_helper code, internal_fn *reduc_fn)
3277 : {
3278 2283305 : if (code.is_tree_code ())
3279 2283247 : switch (tree_code (code))
3280 : {
3281 14226 : case MAX_EXPR:
3282 14226 : *reduc_fn = IFN_REDUC_MAX;
3283 14226 : return true;
3284 :
3285 49470 : case MIN_EXPR:
3286 49470 : *reduc_fn = IFN_REDUC_MIN;
3287 49470 : return true;
3288 :
3289 1250892 : case PLUS_EXPR:
3290 1250892 : *reduc_fn = IFN_REDUC_PLUS;
3291 1250892 : return true;
3292 :
3293 250937 : case BIT_AND_EXPR:
3294 250937 : *reduc_fn = IFN_REDUC_AND;
3295 250937 : return true;
3296 :
3297 270671 : case BIT_IOR_EXPR:
3298 270671 : *reduc_fn = IFN_REDUC_IOR;
3299 270671 : return true;
3300 :
3301 44360 : case BIT_XOR_EXPR:
3302 44360 : *reduc_fn = IFN_REDUC_XOR;
3303 44360 : return true;
3304 :
3305 402691 : case MULT_EXPR:
3306 402691 : case MINUS_EXPR:
3307 402691 : *reduc_fn = IFN_LAST;
3308 402691 : return true;
3309 :
3310 : default:
3311 : return false;
3312 : }
3313 : else
3314 58 : switch (combined_fn (code))
3315 : {
3316 34 : CASE_CFN_FMAX:
3317 34 : *reduc_fn = IFN_REDUC_FMAX;
3318 34 : return true;
3319 :
3320 24 : CASE_CFN_FMIN:
3321 24 : *reduc_fn = IFN_REDUC_FMIN;
3322 24 : return true;
3323 :
3324 : default:
3325 : return false;
3326 : }
3327 : }
3328 :
3329 : /* Set *SBOOL_FN to the corresponding function working on vector masks
3330 : for REDUC_FN. Return true if that exists, false otherwise. */
3331 :
3332 : static bool
3333 0 : sbool_reduction_fn_for_fn (internal_fn reduc_fn, internal_fn *sbool_fn)
3334 : {
3335 0 : switch (reduc_fn)
3336 : {
3337 0 : case IFN_REDUC_AND:
3338 0 : *sbool_fn = IFN_REDUC_SBOOL_AND;
3339 0 : return true;
3340 0 : case IFN_REDUC_IOR:
3341 0 : *sbool_fn = IFN_REDUC_SBOOL_IOR;
3342 0 : return true;
3343 0 : case IFN_REDUC_XOR:
3344 0 : *sbool_fn = IFN_REDUC_SBOOL_XOR;
3345 0 : return true;
3346 : default:
3347 : return false;
3348 : }
3349 : }
3350 :
3351 : /* If there is a neutral value X such that a reduction would not be affected
3352 : by the introduction of additional X elements, return that X, otherwise
3353 : return null. CODE is the code of the reduction and SCALAR_TYPE is type
3354 : of the scalar elements. If the reduction has just a single initial value
3355 : then INITIAL_VALUE is that value, otherwise it is null.
3356 : If AS_INITIAL is TRUE the value is supposed to be used as initial value.
3357 : In that case no signed zero is returned. */
3358 :
3359 : tree
3360 78398 : neutral_op_for_reduction (tree scalar_type, code_helper code,
3361 : tree initial_value, bool as_initial)
3362 : {
3363 78398 : if (code.is_tree_code ())
3364 78340 : switch (tree_code (code))
3365 : {
3366 13976 : case DOT_PROD_EXPR:
3367 13976 : case SAD_EXPR:
3368 13976 : case MINUS_EXPR:
3369 13976 : case BIT_IOR_EXPR:
3370 13976 : case BIT_XOR_EXPR:
3371 13976 : return build_zero_cst (scalar_type);
3372 58031 : case WIDEN_SUM_EXPR:
3373 58031 : case PLUS_EXPR:
3374 58031 : if (!as_initial && HONOR_SIGNED_ZEROS (scalar_type))
3375 92 : return build_real (scalar_type, dconstm0);
3376 : else
3377 57939 : return build_zero_cst (scalar_type);
3378 :
3379 2254 : case MULT_EXPR:
3380 2254 : return build_one_cst (scalar_type);
3381 :
3382 1573 : case BIT_AND_EXPR:
3383 1573 : return build_all_ones_cst (scalar_type);
3384 :
3385 : case MAX_EXPR:
3386 : case MIN_EXPR:
3387 : return initial_value;
3388 :
3389 453 : default:
3390 453 : return NULL_TREE;
3391 : }
3392 : else
3393 58 : switch (combined_fn (code))
3394 : {
3395 : CASE_CFN_FMIN:
3396 : CASE_CFN_FMAX:
3397 : return initial_value;
3398 :
3399 0 : default:
3400 0 : return NULL_TREE;
3401 : }
3402 : }
3403 :
3404 : /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
3405 : STMT is printed with a message MSG. */
3406 :
3407 : static void
3408 578 : report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
3409 : {
3410 578 : dump_printf_loc (msg_type, vect_location, "%s%G", msg, stmt);
3411 578 : }
3412 :
3413 : /* Return true if we need an in-order reduction for operation CODE
3414 : on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
3415 : overflow must wrap. */
3416 :
3417 : bool
3418 363553 : needs_fold_left_reduction_p (tree type, code_helper code)
3419 : {
3420 : /* CHECKME: check for !flag_finite_math_only too? */
3421 363553 : if (SCALAR_FLOAT_TYPE_P (type))
3422 : {
3423 101498 : if (code.is_tree_code ())
3424 101444 : switch (tree_code (code))
3425 : {
3426 : case MIN_EXPR:
3427 : case MAX_EXPR:
3428 : return false;
3429 :
3430 100868 : default:
3431 100868 : return !flag_associative_math;
3432 : }
3433 : else
3434 54 : switch (combined_fn (code))
3435 : {
3436 : CASE_CFN_FMIN:
3437 : CASE_CFN_FMAX:
3438 : return false;
3439 :
3440 2 : default:
3441 2 : return !flag_associative_math;
3442 : }
3443 : }
3444 :
3445 262055 : if (INTEGRAL_TYPE_P (type))
3446 261945 : return (!code.is_tree_code ()
3447 261945 : || !operation_no_trapping_overflow (type, tree_code (code)));
3448 :
3449 110 : if (SAT_FIXED_POINT_TYPE_P (type))
3450 : return true;
3451 :
3452 : return false;
3453 : }
3454 :
3455 : /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
3456 : has a handled computation expression. Store the main reduction
3457 : operation in *CODE. */
3458 :
3459 : static bool
3460 103529 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3461 : tree loop_arg, code_helper *code,
3462 : vec<std::pair<ssa_op_iter, use_operand_p> > &path,
3463 : bool inner_loop_of_double_reduc)
3464 : {
3465 103529 : auto_bitmap visited;
3466 103529 : tree lookfor = PHI_RESULT (phi);
3467 103529 : ssa_op_iter curri;
3468 103529 : use_operand_p curr = op_iter_init_phiuse (&curri, phi, SSA_OP_USE);
3469 215253 : while (USE_FROM_PTR (curr) != loop_arg)
3470 8195 : curr = op_iter_next_use (&curri);
3471 103529 : curri.i = curri.numops;
3472 957021 : do
3473 : {
3474 957021 : path.safe_push (std::make_pair (curri, curr));
3475 957021 : tree use = USE_FROM_PTR (curr);
3476 957021 : if (use == lookfor)
3477 : break;
3478 853987 : gimple *def = SSA_NAME_DEF_STMT (use);
3479 853987 : if (gimple_nop_p (def)
3480 853987 : || ! flow_bb_inside_loop_p (loop, gimple_bb (def)))
3481 : {
3482 241077 : pop:
3483 720545 : do
3484 : {
3485 720545 : std::pair<ssa_op_iter, use_operand_p> x = path.pop ();
3486 720545 : curri = x.first;
3487 720545 : curr = x.second;
3488 788741 : do
3489 788741 : curr = op_iter_next_use (&curri);
3490 : /* Skip already visited or non-SSA operands (from iterating
3491 : over PHI args). */
3492 : while (curr != NULL_USE_OPERAND_P
3493 1097519 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3494 272566 : || ! bitmap_set_bit (visited,
3495 272566 : SSA_NAME_VERSION
3496 : (USE_FROM_PTR (curr)))));
3497 : }
3498 1441090 : while (curr == NULL_USE_OPERAND_P && ! path.is_empty ());
3499 241077 : if (curr == NULL_USE_OPERAND_P)
3500 : break;
3501 : }
3502 : else
3503 : {
3504 718094 : if (gimple_code (def) == GIMPLE_PHI)
3505 73367 : curr = op_iter_init_phiuse (&curri, as_a <gphi *>(def), SSA_OP_USE);
3506 : else
3507 644727 : curr = op_iter_init_use (&curri, def, SSA_OP_USE);
3508 : while (curr != NULL_USE_OPERAND_P
3509 858414 : && (TREE_CODE (USE_FROM_PTR (curr)) != SSA_NAME
3510 747819 : || ! bitmap_set_bit (visited,
3511 747819 : SSA_NAME_VERSION
3512 : (USE_FROM_PTR (curr)))))
3513 140320 : curr = op_iter_next_use (&curri);
3514 718094 : if (curr == NULL_USE_OPERAND_P)
3515 105184 : goto pop;
3516 : }
3517 : }
3518 : while (1);
3519 103529 : if (dump_file && (dump_flags & TDF_DETAILS))
3520 : {
3521 4193 : dump_printf_loc (MSG_NOTE, loc, "reduction path: ");
3522 4193 : unsigned i;
3523 4193 : std::pair<ssa_op_iter, use_operand_p> *x;
3524 14292 : FOR_EACH_VEC_ELT (path, i, x)
3525 10099 : dump_printf (MSG_NOTE, "%T ", USE_FROM_PTR (x->second));
3526 4193 : dump_printf (MSG_NOTE, "\n");
3527 : }
3528 :
3529 : /* Check whether the reduction path detected is valid. */
3530 103529 : bool fail = path.length () == 0;
3531 103529 : bool neg = false;
3532 103529 : int sign = -1;
3533 103529 : *code = ERROR_MARK;
3534 226207 : for (unsigned i = 1; i < path.length (); ++i)
3535 : {
3536 126307 : gimple *use_stmt = USE_STMT (path[i].second);
3537 126307 : gimple_match_op op;
3538 126307 : if (!gimple_extract_op (use_stmt, &op))
3539 : {
3540 : fail = true;
3541 3629 : break;
3542 : }
3543 125391 : unsigned int opi = op.num_ops;
3544 125391 : if (gassign *assign = dyn_cast<gassign *> (use_stmt))
3545 : {
3546 : /* The following make sure we can compute the operand index
3547 : easily plus it mostly disallows chaining via COND_EXPR condition
3548 : operands. */
3549 193830 : for (opi = 0; opi < op.num_ops; ++opi)
3550 192800 : if (gimple_assign_rhs1_ptr (assign) + opi == path[i].second->use)
3551 : break;
3552 : }
3553 6238 : else if (gcall *call = dyn_cast<gcall *> (use_stmt))
3554 : {
3555 12485 : for (opi = 0; opi < op.num_ops; ++opi)
3556 12485 : if (gimple_call_arg_ptr (call, opi) == path[i].second->use)
3557 : break;
3558 : }
3559 125391 : if (opi == op.num_ops)
3560 : {
3561 : fail = true;
3562 : break;
3563 : }
3564 124361 : op.code = canonicalize_code (op.code, op.type);
3565 124361 : if (op.code == MINUS_EXPR)
3566 : {
3567 5691 : op.code = PLUS_EXPR;
3568 : /* Track whether we negate the reduction value each iteration. */
3569 5691 : if (op.ops[1] == op.ops[opi])
3570 34 : neg = ! neg;
3571 : }
3572 118670 : else if (op.code == IFN_COND_SUB)
3573 : {
3574 9 : op.code = IFN_COND_ADD;
3575 : /* Track whether we negate the reduction value each iteration. */
3576 9 : if (op.ops[2] == op.ops[opi])
3577 0 : neg = ! neg;
3578 : }
3579 : /* For an FMA the reduction code is the PLUS if the addition chain
3580 : is the reduction. */
3581 118661 : else if (op.code == IFN_FMA && opi == 2)
3582 33 : op.code = PLUS_EXPR;
3583 124361 : if (CONVERT_EXPR_CODE_P (op.code)
3584 124361 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
3585 : ;
3586 118753 : else if (*code == ERROR_MARK)
3587 : {
3588 101198 : *code = op.code;
3589 101198 : sign = TYPE_SIGN (op.type);
3590 : }
3591 17555 : else if (op.code != *code)
3592 : {
3593 : fail = true;
3594 : break;
3595 : }
3596 16125 : else if ((op.code == MIN_EXPR
3597 15969 : || op.code == MAX_EXPR)
3598 16138 : && sign != TYPE_SIGN (op.type))
3599 : {
3600 : fail = true;
3601 : break;
3602 : }
3603 : /* Check there's only a single stmt the op is used on. For the
3604 : not value-changing tail and the last stmt allow out-of-loop uses,
3605 : but not when this is the inner loop of a double reduction.
3606 : ??? We could relax this and handle arbitrary live stmts by
3607 : forcing a scalar epilogue for example. */
3608 122928 : imm_use_iterator imm_iter;
3609 122928 : use_operand_p use_p;
3610 122928 : gimple *op_use_stmt;
3611 122928 : unsigned cnt = 0;
3612 129131 : bool cond_fn_p = op.code.is_internal_fn ()
3613 6203 : && (conditional_internal_fn_code (internal_fn (op.code))
3614 122928 : != ERROR_MARK);
3615 :
3616 295389 : FOR_EACH_IMM_USE_STMT (op_use_stmt, imm_iter, op.ops[opi])
3617 : {
3618 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we should
3619 : have op1 twice (once as definition, once as else) in the same
3620 : operation. Enforce this. */
3621 172461 : if (cond_fn_p && op_use_stmt == use_stmt)
3622 : {
3623 6121 : gcall *call = as_a<gcall *> (use_stmt);
3624 6121 : unsigned else_pos
3625 6121 : = internal_fn_else_index (internal_fn (op.code));
3626 6121 : if (gimple_call_arg (call, else_pos) != op.ops[opi])
3627 : {
3628 : fail = true;
3629 : break;
3630 : }
3631 30605 : for (unsigned int j = 0; j < gimple_call_num_args (call); ++j)
3632 : {
3633 24484 : if (j == else_pos)
3634 6121 : continue;
3635 18363 : if (gimple_call_arg (call, j) == op.ops[opi])
3636 6121 : cnt++;
3637 : }
3638 : }
3639 166340 : else if (!is_gimple_debug (op_use_stmt)
3640 166340 : && ((*code != ERROR_MARK || inner_loop_of_double_reduc)
3641 2830 : || flow_bb_inside_loop_p (loop,
3642 2830 : gimple_bb (op_use_stmt))))
3643 234261 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
3644 117135 : cnt++;
3645 122928 : }
3646 :
3647 122928 : if (cnt != 1)
3648 : {
3649 : fail = true;
3650 : break;
3651 : }
3652 : }
3653 107664 : return ! fail && ! neg && *code != ERROR_MARK;
3654 103529 : }
3655 :
3656 : bool
3657 21 : check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
3658 : tree loop_arg, enum tree_code code)
3659 : {
3660 21 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3661 21 : code_helper code_;
3662 21 : return (check_reduction_path (loc, loop, phi, loop_arg, &code_, path, false)
3663 21 : && code_ == code);
3664 21 : }
3665 :
3666 :
3667 :
3668 : /* Function vect_is_simple_reduction
3669 :
3670 : (1) Detect a cross-iteration def-use cycle that represents a simple
3671 : reduction computation. We look for the following pattern:
3672 :
3673 : loop_header:
3674 : a1 = phi < a0, a2 >
3675 : a3 = ...
3676 : a2 = operation (a3, a1)
3677 :
3678 : or
3679 :
3680 : a3 = ...
3681 : loop_header:
3682 : a1 = phi < a0, a2 >
3683 : a2 = operation (a3, a1)
3684 :
3685 : such that:
3686 : 1. operation is commutative and associative and it is safe to
3687 : change the order of the computation
3688 : 2. no uses for a2 in the loop (a2 is used out of the loop)
3689 : 3. no uses of a1 in the loop besides the reduction operation
3690 : 4. no uses of a1 outside the loop.
3691 :
3692 : Conditions 1,4 are tested here.
3693 : Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
3694 :
3695 : (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
3696 : nested cycles.
3697 :
3698 : (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
3699 : reductions:
3700 :
3701 : a1 = phi < a0, a2 >
3702 : inner loop (def of a3)
3703 : a2 = phi < a3 >
3704 :
3705 : (4) Detect condition expressions, ie:
3706 : for (int i = 0; i < N; i++)
3707 : if (a[i] < val)
3708 : ret_val = a[i];
3709 :
3710 : */
3711 :
3712 : static stmt_vec_info
3713 170280 : vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
3714 : gphi **double_reduc)
3715 : {
3716 170280 : gphi *phi = as_a <gphi *> (phi_info->stmt);
3717 170280 : gimple *phi_use_stmt = NULL;
3718 170280 : imm_use_iterator imm_iter;
3719 170280 : use_operand_p use_p;
3720 :
3721 : /* When double_reduc is NULL we are testing the inner loop of a
3722 : double reduction. */
3723 170280 : bool inner_loop_of_double_reduc = double_reduc == NULL;
3724 170280 : if (double_reduc)
3725 169161 : *double_reduc = NULL;
3726 170280 : STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION;
3727 :
3728 170280 : tree phi_name = PHI_RESULT (phi);
3729 : /* ??? If there are no uses of the PHI result the inner loop reduction
3730 : won't be detected as possibly double-reduction by vectorizable_reduction
3731 : because that tries to walk the PHI arg from the preheader edge which
3732 : can be constant. See PR60382. */
3733 170280 : if (has_zero_uses (phi_name))
3734 : return NULL;
3735 170148 : class loop *loop = (gimple_bb (phi))->loop_father;
3736 170148 : unsigned nphi_def_loop_uses = 0;
3737 480874 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, phi_name)
3738 : {
3739 323373 : gimple *use_stmt = USE_STMT (use_p);
3740 323373 : if (is_gimple_debug (use_stmt))
3741 93564 : continue;
3742 :
3743 229809 : if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3744 : {
3745 12647 : if (dump_enabled_p ())
3746 31 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3747 : "intermediate value used outside loop.\n");
3748 :
3749 12647 : return NULL;
3750 : }
3751 :
3752 : /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
3753 : op1 twice (once as definition, once as else) in the same operation.
3754 : Only count it as one. */
3755 217162 : if (use_stmt != phi_use_stmt)
3756 : {
3757 210681 : nphi_def_loop_uses++;
3758 210681 : phi_use_stmt = use_stmt;
3759 : }
3760 12647 : }
3761 :
3762 157501 : tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
3763 157501 : if (TREE_CODE (latch_def) != SSA_NAME)
3764 : {
3765 1497 : if (dump_enabled_p ())
3766 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3767 : "reduction: not ssa_name: %T\n", latch_def);
3768 : return NULL;
3769 : }
3770 :
3771 156004 : stmt_vec_info def_stmt_info = loop_info->lookup_def (latch_def);
3772 156004 : if (!def_stmt_info
3773 156004 : || !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt_info->stmt)))
3774 : return NULL;
3775 :
3776 155830 : bool nested_in_vect_loop
3777 155830 : = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info), loop);
3778 155830 : unsigned nlatch_def_loop_uses = 0;
3779 155830 : auto_vec<gphi *, 3> lcphis;
3780 616997 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, latch_def)
3781 : {
3782 461167 : gimple *use_stmt = USE_STMT (use_p);
3783 461167 : if (is_gimple_debug (use_stmt))
3784 144689 : continue;
3785 316478 : if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
3786 198378 : nlatch_def_loop_uses++;
3787 : else
3788 : /* We can have more than one loop-closed PHI. */
3789 118100 : lcphis.safe_push (as_a <gphi *> (use_stmt));
3790 155830 : }
3791 :
3792 : /* If we are vectorizing an inner reduction we are executing that
3793 : in the original order only in case we are not dealing with a
3794 : double reduction. */
3795 155830 : if (nested_in_vect_loop && !inner_loop_of_double_reduc)
3796 : {
3797 2313 : if (dump_enabled_p ())
3798 434 : report_vect_op (MSG_NOTE, def_stmt_info->stmt,
3799 : "detected nested cycle: ");
3800 : return def_stmt_info;
3801 : }
3802 :
3803 : /* When the inner loop of a double reduction ends up with more than
3804 : one loop-closed PHI we have failed to classify alternate such
3805 : PHIs as double reduction, leading to wrong code. See PR103237. */
3806 154624 : if (inner_loop_of_double_reduc && lcphis.length () != 1)
3807 : {
3808 1 : if (dump_enabled_p ())
3809 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3810 : "unhandle double reduction\n");
3811 : return NULL;
3812 : }
3813 :
3814 : /* If this isn't a nested cycle or if the nested cycle reduction value
3815 : is used outside of the inner loop we cannot handle uses of the reduction
3816 : value. */
3817 153516 : if (nlatch_def_loop_uses > 1 || nphi_def_loop_uses > 1)
3818 : {
3819 48633 : if (dump_enabled_p ())
3820 412 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3821 : "reduction used in loop.\n");
3822 : return NULL;
3823 : }
3824 :
3825 : /* If DEF_STMT is a phi node itself, we expect it to have a single argument
3826 : defined in the inner loop. */
3827 104883 : if (gphi *def_stmt = dyn_cast <gphi *> (def_stmt_info->stmt))
3828 : {
3829 1375 : tree op1 = PHI_ARG_DEF (def_stmt, 0);
3830 1375 : if (gimple_phi_num_args (def_stmt) != 1
3831 1375 : || TREE_CODE (op1) != SSA_NAME)
3832 : {
3833 92 : if (dump_enabled_p ())
3834 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3835 : "unsupported phi node definition.\n");
3836 :
3837 : return NULL;
3838 : }
3839 :
3840 : /* Verify there is an inner cycle composed of the PHI phi_use_stmt
3841 : and the latch definition op1. */
3842 1283 : gimple *def1 = SSA_NAME_DEF_STMT (op1);
3843 1283 : if (gimple_bb (def1)
3844 1283 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
3845 1283 : && loop->inner
3846 1229 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
3847 1229 : && (is_gimple_assign (def1) || is_gimple_call (def1))
3848 1220 : && is_a <gphi *> (phi_use_stmt)
3849 1208 : && flow_bb_inside_loop_p (loop->inner, gimple_bb (phi_use_stmt))
3850 1208 : && (op1 == PHI_ARG_DEF_FROM_EDGE (phi_use_stmt,
3851 : loop_latch_edge (loop->inner)))
3852 158232 : && lcphis.length () == 1)
3853 : {
3854 1119 : if (dump_enabled_p ())
3855 144 : report_vect_op (MSG_NOTE, def_stmt,
3856 : "detected double reduction: ");
3857 :
3858 1119 : *double_reduc = as_a <gphi *> (phi_use_stmt);
3859 1119 : return def_stmt_info;
3860 : }
3861 :
3862 : return NULL;
3863 : }
3864 :
3865 : /* Look for the expression computing latch_def from then loop PHI result. */
3866 103508 : auto_vec<std::pair<ssa_op_iter, use_operand_p> > path;
3867 103508 : code_helper code;
3868 103508 : if (check_reduction_path (vect_location, loop, phi, latch_def, &code,
3869 : path, inner_loop_of_double_reduc))
3870 : {
3871 99373 : STMT_VINFO_REDUC_CODE (phi_info) = code;
3872 99373 : if (code == COND_EXPR && !nested_in_vect_loop)
3873 8515 : STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION;
3874 :
3875 : /* Fill in STMT_VINFO_REDUC_IDX. */
3876 99373 : unsigned i;
3877 319638 : for (i = path.length () - 1; i >= 1; --i)
3878 : {
3879 120892 : gimple *stmt = USE_STMT (path[i].second);
3880 120892 : stmt_vec_info stmt_info = loop_info->lookup_stmt (stmt);
3881 120892 : gimple_match_op op;
3882 120892 : if (!gimple_extract_op (stmt, &op))
3883 0 : gcc_unreachable ();
3884 120892 : if (gassign *assign = dyn_cast<gassign *> (stmt))
3885 114674 : STMT_VINFO_REDUC_IDX (stmt_info)
3886 114674 : = path[i].second->use - gimple_assign_rhs1_ptr (assign);
3887 : else
3888 : {
3889 6218 : gcall *call = as_a<gcall *> (stmt);
3890 6218 : STMT_VINFO_REDUC_IDX (stmt_info)
3891 6218 : = path[i].second->use - gimple_call_arg_ptr (call, 0);
3892 : }
3893 : }
3894 99373 : if (dump_enabled_p ())
3895 4179 : dump_printf_loc (MSG_NOTE, vect_location,
3896 : "reduction: detected reduction\n");
3897 :
3898 : return def_stmt_info;
3899 : }
3900 :
3901 4135 : if (dump_enabled_p ())
3902 95 : dump_printf_loc (MSG_NOTE, vect_location,
3903 : "reduction: unknown pattern\n");
3904 :
3905 : return NULL;
3906 259338 : }
3907 :
3908 : /* Estimate the number of peeled epilogue iterations for LOOP_VINFO.
3909 : PEEL_ITERS_PROLOGUE is the number of peeled prologue iterations,
3910 : or -1 if not known. */
3911 :
3912 : static int
3913 509434 : vect_get_peel_iters_epilogue (loop_vec_info loop_vinfo, int peel_iters_prologue)
3914 : {
3915 509434 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
3916 509434 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) || peel_iters_prologue == -1)
3917 : {
3918 212603 : if (dump_enabled_p ())
3919 4097 : dump_printf_loc (MSG_NOTE, vect_location,
3920 : "cost model: epilogue peel iters set to vf/2 "
3921 : "because loop iterations are unknown .\n");
3922 212603 : return assumed_vf / 2;
3923 : }
3924 : else
3925 : {
3926 296831 : int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
3927 296831 : peel_iters_prologue = MIN (niters, peel_iters_prologue);
3928 296831 : int peel_iters_epilogue = (niters - peel_iters_prologue) % assumed_vf;
3929 : /* If we need to peel for gaps, but no peeling is required, we have to
3930 : peel VF iterations. */
3931 296831 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) && !peel_iters_epilogue)
3932 509434 : peel_iters_epilogue = assumed_vf;
3933 : return peel_iters_epilogue;
3934 : }
3935 : }
3936 :
3937 : /* Calculate cost of peeling the scalar loop PEEL_ITERS_PROLOGUE times for
3938 : a prologue and the corresponding times for the epilogue. */
3939 : int
3940 383086 : vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue)
3941 : {
3942 383086 : int retval = 0;
3943 :
3944 383086 : int peel_iters_epilogue
3945 383086 : = vect_get_peel_iters_epilogue (loop_vinfo, peel_iters_prologue);
3946 :
3947 383086 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
3948 : {
3949 : /* If peeled iterations are known but number of scalar loop
3950 : iterations are unknown, count a taken branch per peeled loop. */
3951 145921 : if (peel_iters_prologue > 0)
3952 90156 : retval = builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
3953 145921 : if (peel_iters_epilogue > 0)
3954 145812 : retval += builtin_vectorization_cost (cond_branch_taken, NULL_TREE, 0);
3955 : }
3956 :
3957 766172 : retval += ((peel_iters_prologue + peel_iters_epilogue)
3958 383086 : * loop_vinfo->scalar_costs->body_cost ());
3959 766172 : retval += (((peel_iters_prologue != 0) + (peel_iters_epilogue != 0))
3960 383086 : * loop_vinfo->scalar_costs->outside_cost ());
3961 :
3962 383086 : return retval;
3963 : }
3964 :
3965 : /* Function vect_estimate_min_profitable_iters
3966 :
3967 : Return the number of iterations required for the vector version of the
3968 : loop to be profitable relative to the cost of the scalar version of the
3969 : loop.
3970 :
3971 : *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
3972 : of iterations for vectorization. -1 value means loop vectorization
3973 : is not profitable. This returned value may be used for dynamic
3974 : profitability check.
3975 :
3976 : *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
3977 : for static check against estimated number of iterations. */
3978 :
3979 : static void
3980 143926 : vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
3981 : int *ret_min_profitable_niters,
3982 : int *ret_min_profitable_estimate,
3983 : unsigned *suggested_unroll_factor)
3984 : {
3985 143926 : int min_profitable_iters;
3986 143926 : int min_profitable_estimate;
3987 143926 : int peel_iters_prologue;
3988 143926 : int peel_iters_epilogue;
3989 143926 : unsigned vec_inside_cost = 0;
3990 143926 : int vec_outside_cost = 0;
3991 143926 : unsigned vec_prologue_cost = 0;
3992 143926 : unsigned vec_epilogue_cost = 0;
3993 143926 : int scalar_single_iter_cost = 0;
3994 143926 : int scalar_outside_cost = 0;
3995 143926 : int assumed_vf = vect_vf_for_cost (loop_vinfo);
3996 143926 : int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
3997 143926 : vector_costs *target_cost_data = loop_vinfo->vector_costs;
3998 :
3999 : /* Cost model disabled. */
4000 143926 : if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
4001 : {
4002 17267 : if (dump_enabled_p ())
4003 10962 : dump_printf_loc (MSG_NOTE, vect_location, "cost model disabled.\n");
4004 17267 : *ret_min_profitable_niters = 0;
4005 17267 : *ret_min_profitable_estimate = 0;
4006 17267 : return;
4007 : }
4008 :
4009 : /* Requires loop versioning tests to handle misalignment. */
4010 126659 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
4011 : {
4012 : /* FIXME: Make cost depend on complexity of individual check. */
4013 18 : unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
4014 18 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4015 18 : if (dump_enabled_p ())
4016 2 : dump_printf (MSG_NOTE,
4017 : "cost model: Adding cost of checks for loop "
4018 : "versioning to treat misalignment.\n");
4019 : }
4020 :
4021 : /* Requires loop versioning with alias checks. */
4022 126659 : if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
4023 : {
4024 : /* FIXME: Make cost depend on complexity of individual check. */
4025 7200 : unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
4026 7200 : (void) add_stmt_cost (target_cost_data, len, scalar_stmt, vect_prologue);
4027 7200 : len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
4028 4 : if (len)
4029 : /* Count LEN - 1 ANDs and LEN comparisons. */
4030 4 : (void) add_stmt_cost (target_cost_data, len * 2 - 1,
4031 : scalar_stmt, vect_prologue);
4032 7200 : len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
4033 1263 : if (len)
4034 : {
4035 : /* Count LEN - 1 ANDs and LEN comparisons. */
4036 1263 : unsigned int nstmts = len * 2 - 1;
4037 : /* +1 for each bias that needs adding. */
4038 2526 : for (unsigned int i = 0; i < len; ++i)
4039 1263 : if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
4040 154 : nstmts += 1;
4041 1263 : (void) add_stmt_cost (target_cost_data, nstmts,
4042 : scalar_stmt, vect_prologue);
4043 : }
4044 7200 : if (dump_enabled_p ())
4045 32 : dump_printf (MSG_NOTE,
4046 : "cost model: Adding cost of checks for loop "
4047 : "versioning aliasing.\n");
4048 : }
4049 :
4050 : /* Requires loop versioning with niter checks. */
4051 126659 : if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
4052 : {
4053 : /* FIXME: Make cost depend on complexity of individual check. */
4054 765 : (void) add_stmt_cost (target_cost_data, 1, vector_stmt,
4055 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4056 765 : if (dump_enabled_p ())
4057 1 : dump_printf (MSG_NOTE,
4058 : "cost model: Adding cost of checks for loop "
4059 : "versioning niters.\n");
4060 : }
4061 :
4062 126659 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4063 7977 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4064 : vect_prologue);
4065 :
4066 : /* Count statements in scalar loop. Using this as scalar cost for a single
4067 : iteration for now.
4068 :
4069 : TODO: Add outer loop support.
4070 :
4071 : TODO: Consider assigning different costs to different scalar
4072 : statements. */
4073 :
4074 126659 : scalar_single_iter_cost = loop_vinfo->scalar_costs->total_cost ();
4075 :
4076 : /* Add additional cost for the peeled instructions in prologue and epilogue
4077 : loop. (For fully-masked loops there will be no peeling.)
4078 :
4079 : FORNOW: If we don't know the value of peel_iters for prologue or epilogue
4080 : at compile-time - we assume it's vf/2 (the worst would be vf-1).
4081 :
4082 : TODO: Build an expression that represents peel_iters for prologue and
4083 : epilogue to be used in a run-time test. */
4084 :
4085 126659 : bool prologue_need_br_taken_cost = false;
4086 126659 : bool prologue_need_br_not_taken_cost = false;
4087 :
4088 : /* Calculate peel_iters_prologue. */
4089 126659 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
4090 : peel_iters_prologue = 0;
4091 126659 : else if (npeel < 0)
4092 : {
4093 282 : peel_iters_prologue = assumed_vf / 2;
4094 282 : if (dump_enabled_p ())
4095 11 : dump_printf (MSG_NOTE, "cost model: "
4096 : "prologue peel iters set to vf/2.\n");
4097 :
4098 : /* If peeled iterations are unknown, count a taken branch and a not taken
4099 : branch per peeled loop. Even if scalar loop iterations are known,
4100 : vector iterations are not known since peeled prologue iterations are
4101 : not known. Hence guards remain the same. */
4102 : prologue_need_br_taken_cost = true;
4103 : prologue_need_br_not_taken_cost = true;
4104 : }
4105 : else
4106 : {
4107 126377 : peel_iters_prologue = npeel;
4108 126377 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_prologue > 0)
4109 : /* If peeled iterations are known but number of scalar loop
4110 : iterations are unknown, count a taken branch per peeled loop. */
4111 126659 : prologue_need_br_taken_cost = true;
4112 : }
4113 :
4114 126659 : bool epilogue_need_br_taken_cost = false;
4115 126659 : bool epilogue_need_br_not_taken_cost = false;
4116 :
4117 : /* Calculate peel_iters_epilogue. */
4118 126659 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4119 : /* We need to peel exactly one iteration for gaps. */
4120 29 : peel_iters_epilogue = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
4121 126630 : else if (npeel < 0)
4122 : {
4123 : /* If peeling for alignment is unknown, loop bound of main loop
4124 : becomes unknown. */
4125 282 : peel_iters_epilogue = assumed_vf / 2;
4126 282 : if (dump_enabled_p ())
4127 11 : dump_printf (MSG_NOTE, "cost model: "
4128 : "epilogue peel iters set to vf/2 because "
4129 : "peeling for alignment is unknown.\n");
4130 :
4131 : /* See the same reason above in peel_iters_prologue calculation. */
4132 : epilogue_need_br_taken_cost = true;
4133 : epilogue_need_br_not_taken_cost = true;
4134 : }
4135 : else
4136 : {
4137 126348 : peel_iters_epilogue = vect_get_peel_iters_epilogue (loop_vinfo, npeel);
4138 126348 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) && peel_iters_epilogue > 0)
4139 : /* If peeled iterations are known but number of scalar loop
4140 : iterations are unknown, count a taken branch per peeled loop. */
4141 126659 : epilogue_need_br_taken_cost = true;
4142 : }
4143 :
4144 : /* The way we cummulate peeling costs into the vector prologue/epilogue
4145 : cost is a bit awkward given we cannot reuse scalar_costs which is
4146 : already computed and also because it cannot take into account any
4147 : epilogue vectorization we'll carry out in the end. */
4148 :
4149 126659 : stmt_info_for_cost *si;
4150 126659 : int j;
4151 : /* Add costs associated with peel_iters_prologue. */
4152 126659 : if (peel_iters_prologue)
4153 1083 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4154 : {
4155 785 : (void) add_stmt_cost (target_cost_data,
4156 785 : si->count * peel_iters_prologue, si->kind,
4157 : si->stmt_info, si->node, si->vectype,
4158 : si->misalign, vect_prologue);
4159 : }
4160 :
4161 : /* Add costs associated with peel_iters_epilogue. */
4162 126659 : if (peel_iters_epilogue)
4163 392698 : FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
4164 : {
4165 314461 : (void) add_stmt_cost (target_cost_data,
4166 314461 : si->count * peel_iters_epilogue, si->kind,
4167 : si->stmt_info, si->node, si->vectype,
4168 : si->misalign, vect_epilogue);
4169 : }
4170 :
4171 : /* Add possible cond_branch_taken/cond_branch_not_taken cost. */
4172 :
4173 126659 : if (prologue_need_br_taken_cost)
4174 282 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4175 : vect_prologue);
4176 :
4177 126659 : if (prologue_need_br_not_taken_cost)
4178 282 : (void) add_stmt_cost (target_cost_data, 1,
4179 : cond_branch_not_taken, vect_prologue);
4180 :
4181 126659 : if (epilogue_need_br_taken_cost)
4182 66068 : (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
4183 : vect_epilogue);
4184 :
4185 126659 : if (epilogue_need_br_not_taken_cost)
4186 282 : (void) add_stmt_cost (target_cost_data, 1,
4187 : cond_branch_not_taken, vect_epilogue);
4188 :
4189 : /* Take care of special costs for rgroup controls of partial vectors. */
4190 29 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4191 126688 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4192 : == vect_partial_vectors_avx512))
4193 : {
4194 : /* Calculate how many masks we need to generate. */
4195 29 : unsigned int num_masks = 0;
4196 29 : bool need_saturation = false;
4197 121 : for (auto rgm : LOOP_VINFO_MASKS (loop_vinfo).rgc_vec)
4198 34 : if (rgm.type)
4199 : {
4200 29 : unsigned nvectors = rgm.factor;
4201 29 : num_masks += nvectors;
4202 29 : if (TYPE_PRECISION (TREE_TYPE (rgm.compare_type))
4203 29 : < TYPE_PRECISION (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo)))
4204 10 : need_saturation = true;
4205 : }
4206 :
4207 : /* ??? The target isn't able to identify the costs below as
4208 : producing masks so it cannot penaltize cases where we'd run
4209 : out of mask registers for example. */
4210 :
4211 : /* ??? We are also failing to account for smaller vector masks
4212 : we generate by splitting larger masks in vect_get_loop_mask. */
4213 :
4214 : /* In the worst case, we need to generate each mask in the prologue
4215 : and in the loop body. We need one splat per group and one
4216 : compare per mask.
4217 :
4218 : Sometimes the prologue mask will fold to a constant,
4219 : so the actual prologue cost might be smaller. However, it's
4220 : simpler and safer to use the worst-case cost; if this ends up
4221 : being the tie-breaker between vectorizing or not, then it's
4222 : probably better not to vectorize. */
4223 29 : (void) add_stmt_cost (target_cost_data,
4224 : num_masks
4225 29 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4226 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4227 : vect_prologue);
4228 58 : (void) add_stmt_cost (target_cost_data,
4229 : num_masks
4230 58 : + LOOP_VINFO_MASKS (loop_vinfo).rgc_vec.length (),
4231 : vector_stmt, NULL, NULL, NULL_TREE, 0, vect_body);
4232 :
4233 : /* When we need saturation we need it both in the prologue and
4234 : the epilogue. */
4235 29 : if (need_saturation)
4236 : {
4237 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4238 : NULL, NULL, NULL_TREE, 0, vect_prologue);
4239 10 : (void) add_stmt_cost (target_cost_data, 1, scalar_stmt,
4240 : NULL, NULL, NULL_TREE, 0, vect_body);
4241 : }
4242 : }
4243 0 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
4244 126630 : && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
4245 : == vect_partial_vectors_while_ult))
4246 : {
4247 : /* Calculate how many masks we need to generate. */
4248 : unsigned int num_masks = 0;
4249 : rgroup_controls *rgm;
4250 : unsigned int num_vectors_m1;
4251 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec,
4252 : num_vectors_m1, rgm)
4253 0 : if (rgm->type)
4254 0 : num_masks += num_vectors_m1 + 1;
4255 0 : gcc_assert (num_masks > 0);
4256 :
4257 : /* In the worst case, we need to generate each mask in the prologue
4258 : and in the loop body. One of the loop body mask instructions
4259 : replaces the comparison in the scalar loop, and since we don't
4260 : count the scalar comparison against the scalar body, we shouldn't
4261 : count that vector instruction against the vector body either.
4262 :
4263 : Sometimes we can use unpacks instead of generating prologue
4264 : masks and sometimes the prologue mask will fold to a constant,
4265 : so the actual prologue cost might be smaller. However, it's
4266 : simpler and safer to use the worst-case cost; if this ends up
4267 : being the tie-breaker between vectorizing or not, then it's
4268 : probably better not to vectorize. */
4269 0 : (void) add_stmt_cost (target_cost_data, num_masks,
4270 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4271 : vect_prologue);
4272 0 : (void) add_stmt_cost (target_cost_data, num_masks - 1,
4273 : vector_stmt, NULL, NULL, NULL_TREE, 0,
4274 : vect_body);
4275 : }
4276 126630 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
4277 : {
4278 : /* Referring to the functions vect_set_loop_condition_partial_vectors
4279 : and vect_set_loop_controls_directly, we need to generate each
4280 : length in the prologue and in the loop body if required. Although
4281 : there are some possible optimizations, we consider the worst case
4282 : here. */
4283 :
4284 0 : bool niters_known_p = LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo);
4285 0 : signed char partial_load_store_bias
4286 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
4287 0 : bool need_iterate_p
4288 0 : = (!LOOP_VINFO_EPILOGUE_P (loop_vinfo)
4289 0 : && !vect_known_niters_smaller_than_vf (loop_vinfo));
4290 :
4291 : /* Calculate how many statements to be added. */
4292 0 : unsigned int prologue_stmts = 0;
4293 0 : unsigned int body_stmts = 0;
4294 :
4295 0 : rgroup_controls *rgc;
4296 0 : unsigned int num_vectors_m1;
4297 0 : FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), num_vectors_m1, rgc)
4298 0 : if (rgc->type)
4299 : {
4300 : /* May need one SHIFT for nitems_total computation. */
4301 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
4302 0 : if (nitems != 1 && !niters_known_p)
4303 0 : prologue_stmts += 1;
4304 :
4305 : /* May need one MAX and one MINUS for wrap around. */
4306 0 : if (vect_rgroup_iv_might_wrap_p (loop_vinfo, rgc))
4307 0 : prologue_stmts += 2;
4308 :
4309 : /* Need one MAX and one MINUS for each batch limit excepting for
4310 : the 1st one. */
4311 0 : prologue_stmts += num_vectors_m1 * 2;
4312 :
4313 0 : unsigned int num_vectors = num_vectors_m1 + 1;
4314 :
4315 : /* Need to set up lengths in prologue, only one MIN required
4316 : for each since start index is zero. */
4317 0 : prologue_stmts += num_vectors;
4318 :
4319 : /* If we have a non-zero partial load bias, we need one PLUS
4320 : to adjust the load length. */
4321 0 : if (partial_load_store_bias != 0)
4322 0 : body_stmts += 1;
4323 :
4324 0 : unsigned int length_update_cost = 0;
4325 0 : if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo))
4326 : /* For decrement IV style, Each only need a single SELECT_VL
4327 : or MIN since beginning to calculate the number of elements
4328 : need to be processed in current iteration. */
4329 : length_update_cost = 1;
4330 : else
4331 : /* For increment IV stype, Each may need two MINs and one MINUS to
4332 : update lengths in body for next iteration. */
4333 0 : length_update_cost = 3;
4334 :
4335 0 : if (need_iterate_p)
4336 0 : body_stmts += length_update_cost * num_vectors;
4337 : }
4338 :
4339 0 : (void) add_stmt_cost (target_cost_data, prologue_stmts,
4340 : scalar_stmt, vect_prologue);
4341 0 : (void) add_stmt_cost (target_cost_data, body_stmts,
4342 : scalar_stmt, vect_body);
4343 : }
4344 :
4345 : /* FORNOW: The scalar outside cost is incremented in one of the
4346 : following ways:
4347 :
4348 : 1. The vectorizer checks for alignment and aliasing and generates
4349 : a condition that allows dynamic vectorization. A cost model
4350 : check is ANDED with the versioning condition. Hence scalar code
4351 : path now has the added cost of the versioning check.
4352 :
4353 : if (cost > th & versioning_check)
4354 : jmp to vector code
4355 :
4356 : Hence run-time scalar is incremented by not-taken branch cost.
4357 :
4358 : 2. The vectorizer then checks if a prologue is required. If the
4359 : cost model check was not done before during versioning, it has to
4360 : be done before the prologue check.
4361 :
4362 : if (cost <= th)
4363 : prologue = scalar_iters
4364 : if (prologue == 0)
4365 : jmp to vector code
4366 : else
4367 : execute prologue
4368 : if (prologue == num_iters)
4369 : go to exit
4370 :
4371 : Hence the run-time scalar cost is incremented by a taken branch,
4372 : plus a not-taken branch, plus a taken branch cost.
4373 :
4374 : 3. The vectorizer then checks if an epilogue is required. If the
4375 : cost model check was not done before during prologue check, it
4376 : has to be done with the epilogue check.
4377 :
4378 : if (prologue == 0)
4379 : jmp to vector code
4380 : else
4381 : execute prologue
4382 : if (prologue == num_iters)
4383 : go to exit
4384 : vector code:
4385 : if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
4386 : jmp to epilogue
4387 :
4388 : Hence the run-time scalar cost should be incremented by 2 taken
4389 : branches.
4390 :
4391 : TODO: The back end may reorder the BBS's differently and reverse
4392 : conditions/branch directions. Change the estimates below to
4393 : something more reasonable. */
4394 :
4395 : /* If the number of iterations is known and we do not do versioning, we can
4396 : decide whether to vectorize at compile time. Hence the scalar version
4397 : do not carry cost model guard costs. */
4398 59715 : if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
4399 186374 : || LOOP_REQUIRES_VERSIONING (loop_vinfo))
4400 : {
4401 : /* Cost model check occurs at versioning. */
4402 68058 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
4403 7977 : scalar_outside_cost += vect_get_stmt_cost (cond_branch_not_taken);
4404 : else
4405 : {
4406 : /* Cost model check occurs at prologue generation. */
4407 60081 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
4408 155 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken)
4409 155 : + vect_get_stmt_cost (cond_branch_not_taken);
4410 : /* Cost model check occurs at epilogue generation. */
4411 : else
4412 59926 : scalar_outside_cost += 2 * vect_get_stmt_cost (cond_branch_taken);
4413 : }
4414 : }
4415 :
4416 : /* Complete the target-specific cost calculations. */
4417 126659 : loop_vinfo->vector_costs->finish_cost (loop_vinfo->scalar_costs);
4418 126659 : vec_prologue_cost = loop_vinfo->vector_costs->prologue_cost ();
4419 126659 : vec_inside_cost = loop_vinfo->vector_costs->body_cost ();
4420 126659 : vec_epilogue_cost = loop_vinfo->vector_costs->epilogue_cost ();
4421 126659 : if (suggested_unroll_factor)
4422 126269 : *suggested_unroll_factor
4423 126269 : = loop_vinfo->vector_costs->suggested_unroll_factor ();
4424 :
4425 126269 : if (suggested_unroll_factor && *suggested_unroll_factor > 1
4426 419 : && LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) != MAX_VECTORIZATION_FACTOR
4427 0 : && !known_le (LOOP_VINFO_VECT_FACTOR (loop_vinfo) *
4428 : *suggested_unroll_factor,
4429 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo)))
4430 : {
4431 0 : if (dump_enabled_p ())
4432 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4433 : "can't unroll as unrolled vectorization factor larger"
4434 : " than maximum vectorization factor: "
4435 : HOST_WIDE_INT_PRINT_UNSIGNED "\n",
4436 : LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo));
4437 0 : *suggested_unroll_factor = 1;
4438 : }
4439 :
4440 126659 : vec_outside_cost = (int)(vec_prologue_cost + vec_epilogue_cost);
4441 :
4442 126659 : if (dump_enabled_p ())
4443 : {
4444 1129 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
4445 1129 : dump_printf (MSG_NOTE, " Vector inside of loop cost: %d\n",
4446 : vec_inside_cost);
4447 1129 : dump_printf (MSG_NOTE, " Vector prologue cost: %d\n",
4448 : vec_prologue_cost);
4449 1129 : dump_printf (MSG_NOTE, " Vector epilogue cost: %d\n",
4450 : vec_epilogue_cost);
4451 1129 : dump_printf (MSG_NOTE, " Scalar iteration cost: %d\n",
4452 : scalar_single_iter_cost);
4453 1129 : dump_printf (MSG_NOTE, " Scalar outside cost: %d\n",
4454 : scalar_outside_cost);
4455 1129 : dump_printf (MSG_NOTE, " Vector outside cost: %d\n",
4456 : vec_outside_cost);
4457 1129 : dump_printf (MSG_NOTE, " prologue iterations: %d\n",
4458 : peel_iters_prologue);
4459 1129 : dump_printf (MSG_NOTE, " epilogue iterations: %d\n",
4460 : peel_iters_epilogue);
4461 : }
4462 :
4463 : /* Calculate number of iterations required to make the vector version
4464 : profitable, relative to the loop bodies only. The following condition
4465 : must hold true:
4466 : SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
4467 : where
4468 : SIC = scalar iteration cost, VIC = vector iteration cost,
4469 : VOC = vector outside cost, VF = vectorization factor,
4470 : NPEEL = prologue iterations + epilogue iterations,
4471 : SOC = scalar outside cost for run time cost model check. */
4472 :
4473 126659 : int saving_per_viter = (scalar_single_iter_cost * assumed_vf
4474 126659 : - vec_inside_cost);
4475 126659 : if (saving_per_viter <= 0)
4476 : {
4477 24493 : if (LOOP_VINFO_LOOP (loop_vinfo)->force_vectorize)
4478 0 : warning_at (vect_location.get_location_t (), OPT_Wopenmp_simd,
4479 : "vectorization did not happen for a simd loop");
4480 :
4481 24493 : if (dump_enabled_p ())
4482 30 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4483 : "cost model: the vector iteration cost = %d "
4484 : "divided by the scalar iteration cost = %d "
4485 : "is greater or equal to the vectorization factor = %d"
4486 : ".\n",
4487 : vec_inside_cost, scalar_single_iter_cost, assumed_vf);
4488 24493 : *ret_min_profitable_niters = -1;
4489 24493 : *ret_min_profitable_estimate = -1;
4490 24493 : return;
4491 : }
4492 :
4493 : /* ??? The "if" arm is written to handle all cases; see below for what
4494 : we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4495 102166 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4496 : {
4497 : /* Rewriting the condition above in terms of the number of
4498 : vector iterations (vniters) rather than the number of
4499 : scalar iterations (niters) gives:
4500 :
4501 : SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
4502 :
4503 : <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
4504 :
4505 : For integer N, X and Y when X > 0:
4506 :
4507 : N * X > Y <==> N >= (Y /[floor] X) + 1. */
4508 21 : int outside_overhead = (vec_outside_cost
4509 21 : - scalar_single_iter_cost * peel_iters_prologue
4510 21 : - scalar_single_iter_cost * peel_iters_epilogue
4511 : - scalar_outside_cost);
4512 : /* We're only interested in cases that require at least one
4513 : vector iteration. */
4514 21 : int min_vec_niters = 1;
4515 21 : if (outside_overhead > 0)
4516 17 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4517 :
4518 21 : if (dump_enabled_p ())
4519 8 : dump_printf (MSG_NOTE, " Minimum number of vector iterations: %d\n",
4520 : min_vec_niters);
4521 :
4522 21 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4523 : {
4524 : /* Now that we know the minimum number of vector iterations,
4525 : find the minimum niters for which the scalar cost is larger:
4526 :
4527 : SIC * niters > VIC * vniters + VOC - SOC
4528 :
4529 : We know that the minimum niters is no more than
4530 : vniters * VF + NPEEL, but it might be (and often is) less
4531 : than that if a partial vector iteration is cheaper than the
4532 : equivalent scalar code. */
4533 21 : int threshold = (vec_inside_cost * min_vec_niters
4534 21 : + vec_outside_cost
4535 21 : - scalar_outside_cost);
4536 21 : if (threshold <= 0)
4537 : min_profitable_iters = 1;
4538 : else
4539 21 : min_profitable_iters = threshold / scalar_single_iter_cost + 1;
4540 : }
4541 : else
4542 : /* Convert the number of vector iterations into a number of
4543 : scalar iterations. */
4544 0 : min_profitable_iters = (min_vec_niters * assumed_vf
4545 0 : + peel_iters_prologue
4546 : + peel_iters_epilogue);
4547 : }
4548 : else
4549 : {
4550 102145 : min_profitable_iters = ((vec_outside_cost - scalar_outside_cost)
4551 102145 : * assumed_vf
4552 102145 : - vec_inside_cost * peel_iters_prologue
4553 102145 : - vec_inside_cost * peel_iters_epilogue);
4554 102145 : if (min_profitable_iters <= 0)
4555 : min_profitable_iters = 0;
4556 : else
4557 : {
4558 86804 : min_profitable_iters /= saving_per_viter;
4559 :
4560 86804 : if ((scalar_single_iter_cost * assumed_vf * min_profitable_iters)
4561 86804 : <= (((int) vec_inside_cost * min_profitable_iters)
4562 86804 : + (((int) vec_outside_cost - scalar_outside_cost)
4563 : * assumed_vf)))
4564 86804 : min_profitable_iters++;
4565 : }
4566 : }
4567 :
4568 102166 : if (dump_enabled_p ())
4569 1099 : dump_printf (MSG_NOTE,
4570 : " Calculated minimum iters for profitability: %d\n",
4571 : min_profitable_iters);
4572 :
4573 102166 : if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
4574 102145 : && min_profitable_iters < (assumed_vf + peel_iters_prologue))
4575 : /* We want the vectorized loop to execute at least once. */
4576 : min_profitable_iters = assumed_vf + peel_iters_prologue;
4577 23297 : else if (min_profitable_iters < peel_iters_prologue)
4578 : /* For LOOP_VINFO_USING_PARTIAL_VECTORS_P, we need to ensure the
4579 : vectorized loop executes at least once. */
4580 : min_profitable_iters = peel_iters_prologue;
4581 :
4582 102166 : if (dump_enabled_p ())
4583 1099 : dump_printf_loc (MSG_NOTE, vect_location,
4584 : " Runtime profitability threshold = %d\n",
4585 : min_profitable_iters);
4586 :
4587 102166 : *ret_min_profitable_niters = min_profitable_iters;
4588 :
4589 : /* Calculate number of iterations required to make the vector version
4590 : profitable, relative to the loop bodies only.
4591 :
4592 : Non-vectorized variant is SIC * niters and it must win over vector
4593 : variant on the expected loop trip count. The following condition must hold true:
4594 : SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC */
4595 :
4596 102166 : if (vec_outside_cost <= 0)
4597 : min_profitable_estimate = 0;
4598 : /* ??? This "else if" arm is written to handle all cases; see below for
4599 : what we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
4600 91326 : else if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4601 : {
4602 : /* This is a repeat of the code above, but with + SOC rather
4603 : than - SOC. */
4604 21 : int outside_overhead = (vec_outside_cost
4605 21 : - scalar_single_iter_cost * peel_iters_prologue
4606 21 : - scalar_single_iter_cost * peel_iters_epilogue
4607 : + scalar_outside_cost);
4608 21 : int min_vec_niters = 1;
4609 21 : if (outside_overhead > 0)
4610 21 : min_vec_niters = outside_overhead / saving_per_viter + 1;
4611 :
4612 21 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
4613 : {
4614 21 : int threshold = (vec_inside_cost * min_vec_niters
4615 21 : + vec_outside_cost
4616 21 : + scalar_outside_cost);
4617 21 : min_profitable_estimate = threshold / scalar_single_iter_cost + 1;
4618 : }
4619 : else
4620 : min_profitable_estimate = (min_vec_niters * assumed_vf
4621 : + peel_iters_prologue
4622 : + peel_iters_epilogue);
4623 : }
4624 : else
4625 : {
4626 91305 : min_profitable_estimate = ((vec_outside_cost + scalar_outside_cost)
4627 91305 : * assumed_vf
4628 91305 : - vec_inside_cost * peel_iters_prologue
4629 91305 : - vec_inside_cost * peel_iters_epilogue)
4630 91305 : / ((scalar_single_iter_cost * assumed_vf)
4631 : - vec_inside_cost);
4632 : }
4633 102166 : min_profitable_estimate = MAX (min_profitable_estimate, min_profitable_iters);
4634 102166 : if (dump_enabled_p ())
4635 1099 : dump_printf_loc (MSG_NOTE, vect_location,
4636 : " Static estimate profitability threshold = %d\n",
4637 : min_profitable_estimate);
4638 :
4639 102166 : *ret_min_profitable_estimate = min_profitable_estimate;
4640 : }
4641 :
4642 : /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
4643 : vector elements (not bits) for a vector with NELT elements. */
4644 : static void
4645 2116 : calc_vec_perm_mask_for_shift (unsigned int offset, unsigned int nelt,
4646 : vec_perm_builder *sel)
4647 : {
4648 : /* The encoding is a single stepped pattern. Any wrap-around is handled
4649 : by vec_perm_indices. */
4650 2116 : sel->new_vector (nelt, 1, 3);
4651 8464 : for (unsigned int i = 0; i < 3; i++)
4652 6348 : sel->quick_push (i + offset);
4653 2116 : }
4654 :
4655 : /* Checks whether the target supports whole-vector shifts for vectors of mode
4656 : MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
4657 : it supports vec_perm_const with masks for all necessary shift amounts. */
4658 : static bool
4659 13921 : have_whole_vector_shift (machine_mode mode)
4660 : {
4661 13921 : if (can_implement_p (vec_shr_optab, mode))
4662 : return true;
4663 :
4664 : /* Variable-length vectors should be handled via the optab. */
4665 63 : unsigned int nelt;
4666 126 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
4667 : return false;
4668 :
4669 63 : vec_perm_builder sel;
4670 63 : vec_perm_indices indices;
4671 315 : for (unsigned int i = nelt / 2; i >= 1; i /= 2)
4672 : {
4673 252 : calc_vec_perm_mask_for_shift (i, nelt, &sel);
4674 252 : indices.new_vector (sel, 2, nelt);
4675 252 : if (!can_vec_perm_const_p (mode, mode, indices, false))
4676 : return false;
4677 : }
4678 : return true;
4679 63 : }
4680 :
4681 : /* Return true if (a) STMT_INFO is a DOT_PROD_EXPR reduction whose
4682 : multiplication operands have differing signs and (b) we intend
4683 : to emulate the operation using a series of signed DOT_PROD_EXPRs.
4684 : See vect_emulate_mixed_dot_prod for the actual sequence used. */
4685 :
4686 : static bool
4687 2450 : vect_is_emulated_mixed_dot_prod (slp_tree slp_node)
4688 : {
4689 2450 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_node);
4690 2450 : gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
4691 1997 : if (!assign || gimple_assign_rhs_code (assign) != DOT_PROD_EXPR)
4692 : return false;
4693 :
4694 856 : tree rhs1 = gimple_assign_rhs1 (assign);
4695 856 : tree rhs2 = gimple_assign_rhs2 (assign);
4696 856 : if (TYPE_SIGN (TREE_TYPE (rhs1)) == TYPE_SIGN (TREE_TYPE (rhs2)))
4697 : return false;
4698 :
4699 645 : return !directly_supported_p (DOT_PROD_EXPR,
4700 : SLP_TREE_VECTYPE (slp_node),
4701 215 : SLP_TREE_VECTYPE
4702 : (SLP_TREE_CHILDREN (slp_node)[0]),
4703 215 : optab_vector_mixed_sign);
4704 : }
4705 :
4706 : /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
4707 : functions. Design better to avoid maintenance issues. */
4708 :
4709 : /* Function vect_model_reduction_cost.
4710 :
4711 : Models cost for a reduction operation, including the vector ops
4712 : generated within the strip-mine loop in some cases, the initial
4713 : definition before the loop, and the epilogue code that must be generated. */
4714 :
4715 : static void
4716 73009 : vect_model_reduction_cost (loop_vec_info loop_vinfo,
4717 : slp_tree node, internal_fn reduc_fn,
4718 : vect_reduction_type reduction_type,
4719 : int ncopies, stmt_vector_for_cost *cost_vec)
4720 : {
4721 73009 : int prologue_cost = 0, epilogue_cost = 0, inside_cost = 0;
4722 73009 : tree vectype;
4723 73009 : machine_mode mode;
4724 73009 : class loop *loop = NULL;
4725 :
4726 73009 : if (loop_vinfo)
4727 73009 : loop = LOOP_VINFO_LOOP (loop_vinfo);
4728 :
4729 : /* Condition reductions generate two reductions in the loop. */
4730 73009 : if (reduction_type == COND_REDUCTION)
4731 349 : ncopies *= 2;
4732 :
4733 73009 : vectype = SLP_TREE_VECTYPE (node);
4734 73009 : mode = TYPE_MODE (vectype);
4735 73009 : stmt_vec_info orig_stmt_info
4736 73009 : = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
4737 :
4738 73009 : gimple_match_op op;
4739 73009 : if (!gimple_extract_op (orig_stmt_info->stmt, &op))
4740 0 : gcc_unreachable ();
4741 :
4742 73009 : if (reduction_type == EXTRACT_LAST_REDUCTION)
4743 : /* No extra instructions are needed in the prologue. The loop body
4744 : operations are costed in vectorizable_condition. */
4745 : inside_cost = 0;
4746 73009 : else if (reduction_type == FOLD_LEFT_REDUCTION)
4747 : {
4748 : /* No extra instructions needed in the prologue. */
4749 4382 : prologue_cost = 0;
4750 :
4751 4382 : if (reduc_fn != IFN_LAST)
4752 : /* Count one reduction-like operation per vector. */
4753 0 : inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
4754 : node, 0, vect_body);
4755 : else
4756 : {
4757 : /* Use NCOPIES deconstructs and NELEMENTS scalar ops. */
4758 4382 : unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
4759 4382 : inside_cost = record_stmt_cost (cost_vec, ncopies,
4760 : vec_deconstruct, node, 0,
4761 : vect_body);
4762 4382 : inside_cost += record_stmt_cost (cost_vec, nelements,
4763 : scalar_stmt, node, 0,
4764 : vect_body);
4765 : }
4766 : }
4767 : else
4768 : {
4769 : /* Add in the cost of the initial definitions. */
4770 68627 : int prologue_stmts;
4771 68627 : if (reduction_type == COND_REDUCTION)
4772 : /* For cond reductions we have four vectors: initial index, step,
4773 : initial result of the data reduction, initial value of the index
4774 : reduction. */
4775 : prologue_stmts = 4;
4776 : else
4777 : /* We need the initial reduction value. */
4778 68278 : prologue_stmts = 1;
4779 68627 : prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
4780 : scalar_to_vec, node, 0,
4781 : vect_prologue);
4782 : }
4783 :
4784 : /* Determine cost of epilogue code.
4785 :
4786 : We have a reduction operator that will reduce the vector in one statement.
4787 : Also requires scalar extract. */
4788 :
4789 73009 : if (!loop || !nested_in_vect_loop_p (loop, orig_stmt_info))
4790 : {
4791 72825 : if (reduc_fn != IFN_LAST)
4792 : {
4793 52731 : if (reduction_type == COND_REDUCTION)
4794 : {
4795 : /* An EQ stmt and an COND_EXPR stmt. */
4796 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4797 : vector_stmt, node, 0,
4798 : vect_epilogue);
4799 : /* Reduction of the max index and a reduction of the found
4800 : values. */
4801 18 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4802 : vec_to_scalar, node, 0,
4803 : vect_epilogue);
4804 : /* A broadcast of the max value. */
4805 18 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4806 : scalar_to_vec, node, 0,
4807 : vect_epilogue);
4808 : }
4809 : else
4810 : {
4811 52713 : epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
4812 : node, 0, vect_epilogue);
4813 52713 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4814 : vec_to_scalar, node, 0,
4815 : vect_epilogue);
4816 : }
4817 : }
4818 20094 : else if (reduction_type == COND_REDUCTION)
4819 : {
4820 331 : unsigned estimated_nunits = vect_nunits_for_cost (vectype);
4821 : /* Extraction of scalar elements. */
4822 331 : epilogue_cost += record_stmt_cost (cost_vec, 2,
4823 : vec_deconstruct, node, 0,
4824 : vect_epilogue);
4825 : /* Scalar max reductions via COND_EXPR / MAX_EXPR. */
4826 331 : epilogue_cost += record_stmt_cost (cost_vec,
4827 331 : 2 * estimated_nunits - 3,
4828 : scalar_stmt, node, 0,
4829 : vect_epilogue);
4830 : }
4831 19763 : else if (reduction_type == EXTRACT_LAST_REDUCTION
4832 19763 : || reduction_type == FOLD_LEFT_REDUCTION)
4833 : /* No extra instructions need in the epilogue. */
4834 : ;
4835 : else
4836 : {
4837 15381 : int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
4838 15381 : tree bitsize = TYPE_SIZE (op.type);
4839 15381 : int element_bitsize = tree_to_uhwi (bitsize);
4840 15381 : int nelements = vec_size_in_bits / element_bitsize;
4841 :
4842 15381 : if (op.code == COND_EXPR)
4843 31 : op.code = MAX_EXPR;
4844 :
4845 : /* We have a whole vector shift available. */
4846 3154 : if (VECTOR_MODE_P (mode)
4847 15381 : && directly_supported_p (op.code, vectype)
4848 27484 : && have_whole_vector_shift (mode))
4849 : {
4850 : /* Final reduction via vector shifts and the reduction operator.
4851 : Also requires scalar extract. */
4852 36309 : epilogue_cost += record_stmt_cost (cost_vec,
4853 24206 : exact_log2 (nelements) * 2,
4854 : vector_stmt, node, 0,
4855 : vect_epilogue);
4856 12103 : epilogue_cost += record_stmt_cost (cost_vec, 1,
4857 : vec_to_scalar, node, 0,
4858 : vect_epilogue);
4859 : }
4860 : else
4861 : /* Use extracts and reduction op for final reduction. For N
4862 : elements, we have N extracts and N-1 reduction ops. */
4863 3278 : epilogue_cost += record_stmt_cost (cost_vec,
4864 3278 : nelements + nelements - 1,
4865 : vector_stmt, node, 0,
4866 : vect_epilogue);
4867 : }
4868 : }
4869 :
4870 73009 : if (dump_enabled_p ())
4871 3054 : dump_printf (MSG_NOTE,
4872 : "vect_model_reduction_cost: inside_cost = %d, "
4873 : "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost,
4874 : prologue_cost, epilogue_cost);
4875 73009 : }
4876 :
4877 : /* SEQ is a sequence of instructions that initialize the reduction
4878 : described by REDUC_INFO. Emit them in the appropriate place. */
4879 :
4880 : static void
4881 467 : vect_emit_reduction_init_stmts (loop_vec_info loop_vinfo,
4882 : vect_reduc_info reduc_info, gimple *seq)
4883 : {
4884 467 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
4885 : {
4886 : /* When reusing an accumulator from the main loop, we only need
4887 : initialization instructions if the main loop can be skipped.
4888 : In that case, emit the initialization instructions at the end
4889 : of the guard block that does the skip. */
4890 18 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
4891 18 : gcc_assert (skip_edge);
4892 18 : gimple_stmt_iterator gsi = gsi_last_bb (skip_edge->src);
4893 18 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
4894 : }
4895 : else
4896 : {
4897 : /* The normal case: emit the initialization instructions on the
4898 : preheader edge. */
4899 449 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
4900 449 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), seq);
4901 : }
4902 467 : }
4903 :
4904 : /* Get at the initial defs for the reduction PHIs for REDUC_INFO,
4905 : which performs a reduction involving GROUP_SIZE scalar statements.
4906 : NUMBER_OF_VECTORS is the number of vector defs to create. If NEUTRAL_OP
4907 : is nonnull, introducing extra elements of that value will not change the
4908 : result. */
4909 :
4910 : static void
4911 21791 : get_initial_defs_for_reduction (loop_vec_info loop_vinfo,
4912 : vect_reduc_info reduc_info,
4913 : tree vector_type,
4914 : vec<tree> *vec_oprnds,
4915 : unsigned int number_of_vectors,
4916 : unsigned int group_size, tree neutral_op)
4917 : {
4918 21791 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
4919 21791 : unsigned HOST_WIDE_INT nunits;
4920 21791 : unsigned j, number_of_places_left_in_vector;
4921 21791 : unsigned int i;
4922 :
4923 43582 : gcc_assert (group_size == initial_values.length () || neutral_op);
4924 :
4925 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
4926 : created vectors. It is greater than 1 if unrolling is performed.
4927 :
4928 : For example, we have two scalar operands, s1 and s2 (e.g., group of
4929 : strided accesses of size two), while NUNITS is four (i.e., four scalars
4930 : of this type can be packed in a vector). The output vector will contain
4931 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
4932 : will be 2).
4933 :
4934 : If GROUP_SIZE > NUNITS, the scalars will be split into several
4935 : vectors containing the operands.
4936 :
4937 : For example, NUNITS is four as before, and the group size is 8
4938 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
4939 : {s5, s6, s7, s8}. */
4940 :
4941 21791 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
4942 : nunits = group_size;
4943 :
4944 21791 : tree vector_elt_type = TREE_TYPE (vector_type);
4945 21791 : number_of_places_left_in_vector = nunits;
4946 21791 : bool constant_p = true;
4947 21791 : tree_vector_builder elts (vector_type, nunits, 1);
4948 21791 : elts.quick_grow (nunits);
4949 21791 : gimple_seq ctor_seq = NULL;
4950 21791 : if (neutral_op
4951 42981 : && !useless_type_conversion_p (vector_elt_type,
4952 21190 : TREE_TYPE (neutral_op)))
4953 : {
4954 248 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
4955 227 : neutral_op = gimple_build (&ctor_seq, COND_EXPR,
4956 : vector_elt_type,
4957 : neutral_op,
4958 : build_all_ones_cst (vector_elt_type),
4959 : build_zero_cst (vector_elt_type));
4960 : else
4961 21 : neutral_op = gimple_convert (&ctor_seq, vector_elt_type, neutral_op);
4962 : }
4963 204445 : for (j = 0; j < nunits * number_of_vectors; ++j)
4964 : {
4965 182654 : tree op;
4966 182654 : i = j % group_size;
4967 :
4968 : /* Get the def before the loop. In reduction chain we have only
4969 : one initial value. Else we have as many as PHIs in the group. */
4970 182654 : if (i >= initial_values.length () || (j > i && neutral_op))
4971 : op = neutral_op;
4972 : else
4973 : {
4974 51476 : if (!useless_type_conversion_p (vector_elt_type,
4975 25738 : TREE_TYPE (initial_values[i])))
4976 : {
4977 263 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
4978 478 : initial_values[i] = gimple_build (&ctor_seq, COND_EXPR,
4979 : vector_elt_type,
4980 239 : initial_values[i],
4981 : build_all_ones_cst
4982 : (vector_elt_type),
4983 : build_zero_cst
4984 : (vector_elt_type));
4985 : else
4986 48 : initial_values[i] = gimple_convert (&ctor_seq,
4987 : vector_elt_type,
4988 24 : initial_values[i]);
4989 : }
4990 25738 : op = initial_values[i];
4991 : }
4992 :
4993 : /* Create 'vect_ = {op0,op1,...,opn}'. */
4994 182654 : number_of_places_left_in_vector--;
4995 182654 : elts[nunits - number_of_places_left_in_vector - 1] = op;
4996 182654 : if (!CONSTANT_CLASS_P (op))
4997 2497 : constant_p = false;
4998 :
4999 182654 : if (number_of_places_left_in_vector == 0)
5000 : {
5001 23378 : tree init;
5002 46756 : if (constant_p && !neutral_op
5003 46458 : ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type), nunits)
5004 23378 : : known_eq (TYPE_VECTOR_SUBPARTS (vector_type), nunits))
5005 : /* Build the vector directly from ELTS. */
5006 23378 : init = gimple_build_vector (&ctor_seq, &elts);
5007 0 : else if (neutral_op)
5008 : {
5009 : /* Build a vector of the neutral value and shift the
5010 : other elements into place. */
5011 0 : init = gimple_build_vector_from_val (&ctor_seq, vector_type,
5012 : neutral_op);
5013 0 : int k = nunits;
5014 0 : while (k > 0 && operand_equal_p (elts[k - 1], neutral_op))
5015 0 : k -= 1;
5016 0 : while (k > 0)
5017 : {
5018 0 : k -= 1;
5019 0 : init = gimple_build (&ctor_seq, CFN_VEC_SHL_INSERT,
5020 0 : vector_type, init, elts[k]);
5021 : }
5022 : }
5023 : else
5024 : {
5025 : /* First time round, duplicate ELTS to fill the
5026 : required number of vectors. */
5027 0 : duplicate_and_interleave (loop_vinfo, &ctor_seq, vector_type,
5028 : elts, number_of_vectors, *vec_oprnds);
5029 0 : break;
5030 : }
5031 23378 : vec_oprnds->quick_push (init);
5032 :
5033 23378 : number_of_places_left_in_vector = nunits;
5034 23378 : elts.new_vector (vector_type, nunits, 1);
5035 23378 : elts.quick_grow (nunits);
5036 23378 : constant_p = true;
5037 : }
5038 : }
5039 21791 : if (ctor_seq != NULL)
5040 467 : vect_emit_reduction_init_stmts (loop_vinfo, reduc_info, ctor_seq);
5041 21791 : }
5042 :
5043 : vect_reduc_info
5044 218929 : info_for_reduction (loop_vec_info loop_vinfo, slp_tree node)
5045 : {
5046 218929 : gcc_assert (node);
5047 218929 : if (node->cycle_info.id == -1)
5048 : return NULL;
5049 214171 : return loop_vinfo->reduc_infos[node->cycle_info.id];
5050 : }
5051 :
5052 : /* See if LOOP_VINFO is an epilogue loop whose main loop had a reduction that
5053 : REDUC_INFO can build on. Adjust REDUC_INFO and return true if so, otherwise
5054 : return false. */
5055 :
5056 : static bool
5057 21355 : vect_find_reusable_accumulator (loop_vec_info loop_vinfo,
5058 : vect_reduc_info reduc_info, tree vectype)
5059 : {
5060 21355 : loop_vec_info main_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
5061 21355 : if (!main_loop_vinfo)
5062 : return false;
5063 :
5064 4499 : if (VECT_REDUC_INFO_TYPE (reduc_info) != TREE_CODE_REDUCTION)
5065 : return false;
5066 :
5067 : /* We are not set up to handle vector bools when they are not mapped
5068 : to vector integer data types. */
5069 4481 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5070 4533 : && GET_MODE_CLASS (TYPE_MODE (vectype)) != MODE_VECTOR_INT)
5071 : return false;
5072 :
5073 4479 : unsigned int num_phis = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).length ();
5074 4479 : auto_vec<tree, 16> main_loop_results (num_phis);
5075 4479 : auto_vec<tree, 16> initial_values (num_phis);
5076 4479 : if (edge main_loop_edge = loop_vinfo->main_loop_edge)
5077 : {
5078 : /* The epilogue loop can be entered either from the main loop or
5079 : from an earlier guard block. */
5080 4330 : edge skip_edge = loop_vinfo->skip_main_loop_edge;
5081 17340 : for (tree incoming_value : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info))
5082 : {
5083 : /* Look for:
5084 :
5085 : INCOMING_VALUE = phi<MAIN_LOOP_RESULT(main loop),
5086 : INITIAL_VALUE(guard block)>. */
5087 4350 : gcc_assert (TREE_CODE (incoming_value) == SSA_NAME);
5088 :
5089 4350 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (incoming_value));
5090 4350 : gcc_assert (gimple_bb (phi) == main_loop_edge->dest);
5091 :
5092 4350 : tree from_main_loop = PHI_ARG_DEF_FROM_EDGE (phi, main_loop_edge);
5093 4350 : tree from_skip = PHI_ARG_DEF_FROM_EDGE (phi, skip_edge);
5094 :
5095 4350 : main_loop_results.quick_push (from_main_loop);
5096 4350 : initial_values.quick_push (from_skip);
5097 : }
5098 : }
5099 : else
5100 : /* The main loop dominates the epilogue loop. */
5101 149 : main_loop_results.splice (VECT_REDUC_INFO_INITIAL_VALUES (reduc_info));
5102 :
5103 : /* See if the main loop has the kind of accumulator we need. */
5104 4479 : vect_reusable_accumulator *accumulator
5105 4479 : = main_loop_vinfo->reusable_accumulators.get (main_loop_results[0]);
5106 4479 : if (!accumulator
5107 8942 : || num_phis != VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).length ()
5108 13417 : || !std::equal (main_loop_results.begin (), main_loop_results.end (),
5109 : VECT_REDUC_INFO_SCALAR_RESULTS (accumulator->reduc_info).begin ()))
5110 : return false;
5111 :
5112 : /* Handle the case where we can reduce wider vectors to narrower ones. */
5113 4469 : tree old_vectype = TREE_TYPE (accumulator->reduc_input);
5114 4469 : unsigned HOST_WIDE_INT m;
5115 4469 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (old_vectype),
5116 4469 : TYPE_VECTOR_SUBPARTS (vectype), &m))
5117 0 : return false;
5118 : /* Check the intermediate vector types and operations are available. */
5119 4469 : tree prev_vectype = old_vectype;
5120 4469 : poly_uint64 intermediate_nunits = TYPE_VECTOR_SUBPARTS (old_vectype);
5121 13033 : while (known_gt (intermediate_nunits, TYPE_VECTOR_SUBPARTS (vectype)))
5122 : {
5123 4619 : intermediate_nunits = exact_div (intermediate_nunits, 2);
5124 4619 : tree intermediate_vectype = get_related_vectype_for_scalar_type
5125 4619 : (TYPE_MODE (vectype), TREE_TYPE (vectype), intermediate_nunits);
5126 4619 : if (!intermediate_vectype
5127 4619 : || !directly_supported_p (VECT_REDUC_INFO_CODE (reduc_info),
5128 : intermediate_vectype)
5129 8718 : || !can_vec_extract (TYPE_MODE (prev_vectype),
5130 4099 : TYPE_MODE (intermediate_vectype)))
5131 : return false;
5132 : prev_vectype = intermediate_vectype;
5133 : }
5134 :
5135 : /* Non-SLP reductions might apply an adjustment after the reduction
5136 : operation, in order to simplify the initialization of the accumulator.
5137 : If the epilogue loop carries on from where the main loop left off,
5138 : it should apply the same adjustment to the final reduction result.
5139 :
5140 : If the epilogue loop can also be entered directly (rather than via
5141 : the main loop), we need to be able to handle that case in the same way,
5142 : with the same adjustment. (In principle we could add a PHI node
5143 : to select the correct adjustment, but in practice that shouldn't be
5144 : necessary.) */
5145 3945 : tree main_adjustment
5146 3945 : = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (accumulator->reduc_info);
5147 3945 : if (loop_vinfo->main_loop_edge && main_adjustment)
5148 : {
5149 3336 : gcc_assert (num_phis == 1);
5150 3336 : tree initial_value = initial_values[0];
5151 : /* Check that we can use INITIAL_VALUE as the adjustment and
5152 : initialize the accumulator with a neutral value instead. */
5153 3336 : if (!operand_equal_p (initial_value, main_adjustment))
5154 : return false;
5155 3326 : initial_values[0] = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
5156 : }
5157 3935 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info) = main_adjustment;
5158 3935 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).truncate (0);
5159 3935 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).splice (initial_values);
5160 3935 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info) = accumulator;
5161 3935 : return true;
5162 4479 : }
5163 :
5164 : /* Reduce the vector VEC_DEF down to VECTYPE with reduction operation
5165 : CODE emitting stmts before GSI. Returns a vector def of VECTYPE. */
5166 :
5167 : static tree
5168 3979 : vect_create_partial_epilog (tree vec_def, tree vectype, code_helper code,
5169 : gimple_seq *seq)
5170 : {
5171 3979 : gcc_assert (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (vec_def))
5172 : || (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (vec_def)))
5173 : == MODE_VECTOR_INT));
5174 3979 : unsigned nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def)).to_constant ();
5175 3979 : unsigned nunits1 = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
5176 3979 : tree stype = TREE_TYPE (vectype);
5177 3979 : tree new_temp = vec_def;
5178 8101 : while (nunits > nunits1)
5179 : {
5180 4122 : nunits /= 2;
5181 4122 : tree vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
5182 4122 : stype, nunits);
5183 4122 : unsigned int bitsize = tree_to_uhwi (TYPE_SIZE (vectype1));
5184 :
5185 : /* The target has to make sure we support lowpart/highpart
5186 : extraction, either via direct vector extract or through
5187 : an integer mode punning. */
5188 4122 : tree dst1, dst2;
5189 4122 : gimple *epilog_stmt;
5190 4122 : if (convert_optab_handler (vec_extract_optab,
5191 4122 : TYPE_MODE (TREE_TYPE (new_temp)),
5192 4122 : TYPE_MODE (vectype1))
5193 : != CODE_FOR_nothing)
5194 : {
5195 : /* Extract sub-vectors directly once vec_extract becomes
5196 : a conversion optab. */
5197 2606 : dst1 = make_ssa_name (vectype1);
5198 2606 : epilog_stmt
5199 5212 : = gimple_build_assign (dst1, BIT_FIELD_REF,
5200 : build3 (BIT_FIELD_REF, vectype1,
5201 2606 : new_temp, TYPE_SIZE (vectype1),
5202 : bitsize_int (0)));
5203 2606 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5204 2606 : dst2 = make_ssa_name (vectype1);
5205 2606 : epilog_stmt
5206 2606 : = gimple_build_assign (dst2, BIT_FIELD_REF,
5207 : build3 (BIT_FIELD_REF, vectype1,
5208 2606 : new_temp, TYPE_SIZE (vectype1),
5209 2606 : bitsize_int (bitsize)));
5210 2606 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5211 : }
5212 : else
5213 : {
5214 : /* Extract via punning to appropriately sized integer mode
5215 : vector. */
5216 1516 : tree eltype = build_nonstandard_integer_type (bitsize, 1);
5217 1516 : tree etype = build_vector_type (eltype, 2);
5218 3032 : gcc_assert (convert_optab_handler (vec_extract_optab,
5219 : TYPE_MODE (etype),
5220 : TYPE_MODE (eltype))
5221 : != CODE_FOR_nothing);
5222 1516 : tree tem = make_ssa_name (etype);
5223 1516 : epilog_stmt = gimple_build_assign (tem, VIEW_CONVERT_EXPR,
5224 : build1 (VIEW_CONVERT_EXPR,
5225 : etype, new_temp));
5226 1516 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5227 1516 : new_temp = tem;
5228 1516 : tem = make_ssa_name (eltype);
5229 1516 : epilog_stmt
5230 3032 : = gimple_build_assign (tem, BIT_FIELD_REF,
5231 : build3 (BIT_FIELD_REF, eltype,
5232 1516 : new_temp, TYPE_SIZE (eltype),
5233 : bitsize_int (0)));
5234 1516 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5235 1516 : dst1 = make_ssa_name (vectype1);
5236 1516 : epilog_stmt = gimple_build_assign (dst1, VIEW_CONVERT_EXPR,
5237 : build1 (VIEW_CONVERT_EXPR,
5238 : vectype1, tem));
5239 1516 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5240 1516 : tem = make_ssa_name (eltype);
5241 1516 : epilog_stmt
5242 1516 : = gimple_build_assign (tem, BIT_FIELD_REF,
5243 : build3 (BIT_FIELD_REF, eltype,
5244 1516 : new_temp, TYPE_SIZE (eltype),
5245 1516 : bitsize_int (bitsize)));
5246 1516 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5247 1516 : dst2 = make_ssa_name (vectype1);
5248 1516 : epilog_stmt = gimple_build_assign (dst2, VIEW_CONVERT_EXPR,
5249 : build1 (VIEW_CONVERT_EXPR,
5250 : vectype1, tem));
5251 1516 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5252 : }
5253 :
5254 4122 : new_temp = gimple_build (seq, code, vectype1, dst1, dst2);
5255 : }
5256 3979 : if (!useless_type_conversion_p (vectype, TREE_TYPE (new_temp)))
5257 : {
5258 46 : tree dst3 = make_ssa_name (vectype);
5259 46 : gimple *epilog_stmt = gimple_build_assign (dst3, VIEW_CONVERT_EXPR,
5260 : build1 (VIEW_CONVERT_EXPR,
5261 : vectype, new_temp));
5262 46 : gimple_seq_add_stmt_without_update (seq, epilog_stmt);
5263 46 : new_temp = dst3;
5264 : }
5265 :
5266 3979 : return new_temp;
5267 : }
5268 :
5269 : /* Function vect_create_epilog_for_reduction
5270 :
5271 : Create code at the loop-epilog to finalize the result of a reduction
5272 : computation.
5273 :
5274 : STMT_INFO is the scalar reduction stmt that is being vectorized.
5275 : SLP_NODE is an SLP node containing a group of reduction statements. The
5276 : first one in this group is STMT_INFO.
5277 : SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
5278 : REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
5279 : (counting from 0)
5280 : LOOP_EXIT is the edge to update in the merge block. In the case of a single
5281 : exit this edge is always the main loop exit.
5282 :
5283 : This function:
5284 : 1. Completes the reduction def-use cycles.
5285 : 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
5286 : by calling the function specified by REDUC_FN if available, or by
5287 : other means (whole-vector shifts or a scalar loop).
5288 : The function also creates a new phi node at the loop exit to preserve
5289 : loop-closed form, as illustrated below.
5290 :
5291 : The flow at the entry to this function:
5292 :
5293 : loop:
5294 : vec_def = phi <vec_init, null> # REDUCTION_PHI
5295 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5296 : s_loop = scalar_stmt # (scalar) STMT_INFO
5297 : loop_exit:
5298 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5299 : use <s_out0>
5300 : use <s_out0>
5301 :
5302 : The above is transformed by this function into:
5303 :
5304 : loop:
5305 : vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
5306 : VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5307 : s_loop = scalar_stmt # (scalar) STMT_INFO
5308 : loop_exit:
5309 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5310 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5311 : v_out2 = reduce <v_out1>
5312 : s_out3 = extract_field <v_out2, 0>
5313 : s_out4 = adjust_result <s_out3>
5314 : use <s_out4>
5315 : use <s_out4>
5316 : */
5317 :
5318 : static void
5319 22064 : vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
5320 : stmt_vec_info stmt_info,
5321 : slp_tree slp_node,
5322 : slp_instance slp_node_instance,
5323 : edge loop_exit)
5324 : {
5325 22064 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
5326 22064 : code_helper code = VECT_REDUC_INFO_CODE (reduc_info);
5327 22064 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
5328 22064 : tree vectype;
5329 22064 : machine_mode mode;
5330 22064 : basic_block exit_bb;
5331 22064 : gimple *new_phi = NULL, *phi = NULL;
5332 22064 : gimple_stmt_iterator exit_gsi;
5333 22064 : tree new_temp = NULL_TREE, new_name, new_scalar_dest;
5334 22064 : gimple *epilog_stmt = NULL;
5335 22064 : gimple *exit_phi;
5336 22064 : tree def;
5337 22064 : tree orig_name, scalar_result;
5338 22064 : imm_use_iterator imm_iter;
5339 22064 : use_operand_p use_p;
5340 22064 : gimple *use_stmt;
5341 22064 : auto_vec<tree> reduc_inputs;
5342 22064 : int j, i;
5343 22064 : vec<tree> &scalar_results = VECT_REDUC_INFO_SCALAR_RESULTS (reduc_info);
5344 22064 : unsigned int k;
5345 : /* SLP reduction without reduction chain, e.g.,
5346 : # a1 = phi <a2, a0>
5347 : # b1 = phi <b2, b0>
5348 : a2 = operation (a1)
5349 : b2 = operation (b1) */
5350 22064 : const bool slp_reduc = !reduc_info->is_reduc_chain;
5351 22064 : tree induction_index = NULL_TREE;
5352 :
5353 22064 : unsigned int group_size = SLP_TREE_LANES (slp_node);
5354 :
5355 22064 : bool double_reduc = false;
5356 22064 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
5357 22064 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
5358 : {
5359 0 : double_reduc = true;
5360 0 : gcc_assert (slp_reduc);
5361 : }
5362 :
5363 22064 : vectype = VECT_REDUC_INFO_VECTYPE (reduc_info);
5364 22064 : gcc_assert (vectype);
5365 22064 : mode = TYPE_MODE (vectype);
5366 :
5367 22064 : tree induc_val = NULL_TREE;
5368 22064 : tree adjustment_def = NULL;
5369 : /* Optimize: for induction condition reduction, if we can't use zero
5370 : for induc_val, use initial_def. */
5371 22064 : if (VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5372 62 : induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
5373 22002 : else if (double_reduc)
5374 : ;
5375 : else
5376 22002 : adjustment_def = VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info);
5377 :
5378 22064 : stmt_vec_info single_live_out_stmt[] = { stmt_info };
5379 22064 : array_slice<const stmt_vec_info> live_out_stmts = single_live_out_stmt;
5380 22064 : if (slp_reduc)
5381 : /* All statements produce live-out values. */
5382 43664 : live_out_stmts = SLP_TREE_SCALAR_STMTS (slp_node);
5383 :
5384 22064 : unsigned vec_num
5385 22064 : = SLP_TREE_VEC_DEFS (slp_node_instance->reduc_phis).length ();
5386 :
5387 : /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
5388 : which is updated with the current index of the loop for every match of
5389 : the original loop's cond_expr (VEC_STMT). This results in a vector
5390 : containing the last time the condition passed for that vector lane.
5391 : The first match will be a 1 to allow 0 to be used for non-matching
5392 : indexes. If there are no matches at all then the vector will be all
5393 : zeroes.
5394 :
5395 : PR92772: This algorithm is broken for architectures that support
5396 : masked vectors, but do not provide fold_extract_last. */
5397 22064 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION)
5398 : {
5399 99 : gcc_assert (!double_reduc);
5400 99 : auto_vec<std::pair<tree, bool>, 2> ccompares;
5401 99 : slp_tree cond_node = slp_node_instance->root;
5402 207 : while (cond_node != slp_node_instance->reduc_phis)
5403 : {
5404 108 : stmt_vec_info cond_info = SLP_TREE_REPRESENTATIVE (cond_node);
5405 108 : if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR)
5406 : {
5407 108 : gimple *vec_stmt
5408 108 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (cond_node)[0]);
5409 108 : gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
5410 108 : ccompares.safe_push
5411 108 : (std::make_pair (gimple_assign_rhs1 (vec_stmt),
5412 108 : SLP_TREE_REDUC_IDX (cond_node) == 2));
5413 : }
5414 108 : int slp_reduc_idx = SLP_TREE_REDUC_IDX (cond_node);
5415 108 : cond_node = SLP_TREE_CHILDREN (cond_node)[slp_reduc_idx];
5416 : }
5417 99 : gcc_assert (ccompares.length () != 0);
5418 :
5419 99 : tree indx_before_incr, indx_after_incr;
5420 99 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
5421 99 : int scalar_precision
5422 99 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype)));
5423 99 : tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
5424 99 : tree cr_index_vector_type = get_related_vectype_for_scalar_type
5425 99 : (TYPE_MODE (vectype), cr_index_scalar_type,
5426 : TYPE_VECTOR_SUBPARTS (vectype));
5427 :
5428 : /* First we create a simple vector induction variable which starts
5429 : with the values {1,2,3,...} (SERIES_VECT) and increments by the
5430 : vector size (STEP). */
5431 :
5432 : /* Create a {1,2,3,...} vector. */
5433 99 : tree series_vect = build_index_vector (cr_index_vector_type, 1, 1);
5434 :
5435 : /* Create a vector of the step value. */
5436 99 : tree step = build_int_cst (cr_index_scalar_type, nunits_out);
5437 99 : tree vec_step = build_vector_from_val (cr_index_vector_type, step);
5438 :
5439 : /* Create an induction variable. */
5440 99 : gimple_stmt_iterator incr_gsi;
5441 99 : bool insert_after;
5442 99 : vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
5443 : &incr_gsi, &insert_after);
5444 99 : create_iv (series_vect, PLUS_EXPR, vec_step, NULL_TREE, loop, &incr_gsi,
5445 : insert_after, &indx_before_incr, &indx_after_incr);
5446 :
5447 : /* Next create a new phi node vector (NEW_PHI_TREE) which starts
5448 : filled with zeros (VEC_ZERO). */
5449 :
5450 : /* Create a vector of 0s. */
5451 99 : tree zero = build_zero_cst (cr_index_scalar_type);
5452 99 : tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
5453 :
5454 : /* Create a vector phi node. */
5455 99 : tree new_phi_tree = make_ssa_name (cr_index_vector_type);
5456 99 : new_phi = create_phi_node (new_phi_tree, loop->header);
5457 99 : add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
5458 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
5459 :
5460 : /* Now take the condition from the loops original cond_exprs
5461 : and produce a new cond_exprs (INDEX_COND_EXPR) which for
5462 : every match uses values from the induction variable
5463 : (INDEX_BEFORE_INCR) otherwise uses values from the phi node
5464 : (NEW_PHI_TREE).
5465 : Finally, we update the phi (NEW_PHI_TREE) to take the value of
5466 : the new cond_expr (INDEX_COND_EXPR). */
5467 99 : gimple_seq stmts = NULL;
5468 306 : for (int i = ccompares.length () - 1; i != -1; --i)
5469 : {
5470 108 : tree ccompare = ccompares[i].first;
5471 108 : if (ccompares[i].second)
5472 81 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5473 : cr_index_vector_type,
5474 : ccompare,
5475 : indx_before_incr, new_phi_tree);
5476 : else
5477 27 : new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR,
5478 : cr_index_vector_type,
5479 : ccompare,
5480 : new_phi_tree, indx_before_incr);
5481 : }
5482 99 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT);
5483 :
5484 : /* Update the phi with the vec cond. */
5485 99 : induction_index = new_phi_tree;
5486 99 : add_phi_arg (as_a <gphi *> (new_phi), induction_index,
5487 : loop_latch_edge (loop), UNKNOWN_LOCATION);
5488 99 : }
5489 :
5490 : /* 2. Create epilog code.
5491 : The reduction epilog code operates across the elements of the vector
5492 : of partial results computed by the vectorized loop.
5493 : The reduction epilog code consists of:
5494 :
5495 : step 1: compute the scalar result in a vector (v_out2)
5496 : step 2: extract the scalar result (s_out3) from the vector (v_out2)
5497 : step 3: adjust the scalar result (s_out3) if needed.
5498 :
5499 : Step 1 can be accomplished using one the following three schemes:
5500 : (scheme 1) using reduc_fn, if available.
5501 : (scheme 2) using whole-vector shifts, if available.
5502 : (scheme 3) using a scalar loop. In this case steps 1+2 above are
5503 : combined.
5504 :
5505 : The overall epilog code looks like this:
5506 :
5507 : s_out0 = phi <s_loop> # original EXIT_PHI
5508 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5509 : v_out2 = reduce <v_out1> # step 1
5510 : s_out3 = extract_field <v_out2, 0> # step 2
5511 : s_out4 = adjust_result <s_out3> # step 3
5512 :
5513 : (step 3 is optional, and steps 1 and 2 may be combined).
5514 : Lastly, the uses of s_out0 are replaced by s_out4. */
5515 :
5516 :
5517 : /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
5518 : v_out1 = phi <VECT_DEF>
5519 : Store them in NEW_PHIS. */
5520 : /* We need to reduce values in all exits. */
5521 22064 : exit_bb = loop_exit->dest;
5522 22064 : exit_gsi = gsi_after_labels (exit_bb);
5523 22064 : reduc_inputs.create (vec_num);
5524 67789 : for (unsigned i = 0; i < vec_num; i++)
5525 : {
5526 23661 : gimple_seq stmts = NULL;
5527 23661 : def = vect_get_slp_vect_def (slp_node, i);
5528 23661 : tree new_def = copy_ssa_name (def);
5529 23661 : phi = create_phi_node (new_def, exit_bb);
5530 23661 : if (LOOP_VINFO_MAIN_EXIT (loop_vinfo) == loop_exit)
5531 23634 : SET_PHI_ARG_DEF (phi, loop_exit->dest_idx, def);
5532 : else
5533 : {
5534 57 : for (unsigned k = 0; k < gimple_phi_num_args (phi); k++)
5535 30 : SET_PHI_ARG_DEF (phi, k, def);
5536 : }
5537 23661 : new_def = gimple_convert (&stmts, vectype, new_def);
5538 23661 : reduc_inputs.quick_push (new_def);
5539 23661 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5540 : }
5541 :
5542 : /* 2.2 Get the original scalar reduction variable as defined in the loop.
5543 : In case STMT is a "pattern-stmt" (i.e. - it represents a reduction
5544 : pattern), the scalar-def is taken from the original stmt that the
5545 : pattern-stmt (STMT) replaces. */
5546 :
5547 22908 : tree scalar_dest = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5548 22064 : tree scalar_type = TREE_TYPE (scalar_dest);
5549 22064 : scalar_results.truncate (0);
5550 22064 : scalar_results.reserve_exact (group_size);
5551 22064 : new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
5552 :
5553 : /* True if we should implement SLP_REDUC using native reduction operations
5554 : instead of scalar operations. */
5555 22064 : const bool direct_slp_reduc
5556 22064 : = (reduc_fn != IFN_LAST
5557 22064 : && slp_reduc
5558 22064 : && !TYPE_VECTOR_SUBPARTS (vectype).is_constant ());
5559 :
5560 : /* If signed overflow is undefined we might need to perform reduction
5561 : computations in an unsigned type. */
5562 22064 : tree compute_vectype = vectype;
5563 22064 : if (ANY_INTEGRAL_TYPE_P (vectype)
5564 14942 : && TYPE_OVERFLOW_UNDEFINED (vectype)
5565 5497 : && code.is_tree_code ()
5566 27561 : && arith_code_with_undefined_signed_overflow ((tree_code) code))
5567 4059 : compute_vectype = unsigned_type_for (vectype);
5568 :
5569 : /* In case of reduction chain, e.g.,
5570 : # a1 = phi <a3, a0>
5571 : a2 = operation (a1)
5572 : a3 = operation (a2),
5573 :
5574 : we may end up with more than one vector result. Here we reduce them
5575 : to one vector.
5576 :
5577 : The same is true for a SLP reduction, e.g.,
5578 : # a1 = phi <a2, a0>
5579 : # b1 = phi <b2, b0>
5580 : a2 = operation (a1)
5581 : b2 = operation (a2),
5582 :
5583 : where we can end up with more than one vector as well. We can
5584 : easily accumulate vectors when the number of vector elements is
5585 : a multiple of the SLP group size.
5586 :
5587 : The same is true if we couldn't use a single defuse cycle. */
5588 22064 : if ((!slp_reduc
5589 : || direct_slp_reduc
5590 : || (slp_reduc
5591 22064 : && constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype), group_size)))
5592 44128 : && reduc_inputs.length () > 1)
5593 : {
5594 544 : gimple_seq stmts = NULL;
5595 544 : tree single_input = reduc_inputs[0];
5596 544 : if (compute_vectype != vectype)
5597 166 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5598 : compute_vectype, single_input);
5599 1986 : for (k = 1; k < reduc_inputs.length (); k++)
5600 : {
5601 1442 : tree input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5602 1442 : compute_vectype, reduc_inputs[k]);
5603 1442 : single_input = gimple_build (&stmts, code, compute_vectype,
5604 : single_input, input);
5605 : }
5606 544 : if (compute_vectype != vectype)
5607 166 : single_input = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5608 : vectype, single_input);
5609 544 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5610 :
5611 544 : reduc_inputs.truncate (0);
5612 544 : reduc_inputs.safe_push (single_input);
5613 : }
5614 :
5615 22064 : tree orig_reduc_input = reduc_inputs[0];
5616 :
5617 : /* If this loop is an epilogue loop that can be skipped after the
5618 : main loop, we can only share a reduction operation between the
5619 : main loop and the epilogue if we put it at the target of the
5620 : skip edge.
5621 :
5622 : We can still reuse accumulators if this check fails. Doing so has
5623 : the minor(?) benefit of making the epilogue loop's scalar result
5624 : independent of the main loop's scalar result. */
5625 22064 : bool unify_with_main_loop_p = false;
5626 22064 : if (VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
5627 3935 : && loop_vinfo->skip_this_loop_edge
5628 3769 : && single_succ_p (exit_bb)
5629 22081 : && single_succ (exit_bb) == loop_vinfo->skip_this_loop_edge->dest)
5630 : {
5631 17 : unify_with_main_loop_p = true;
5632 :
5633 17 : basic_block reduc_block = loop_vinfo->skip_this_loop_edge->dest;
5634 17 : reduc_inputs[0] = make_ssa_name (vectype);
5635 17 : gphi *new_phi = create_phi_node (reduc_inputs[0], reduc_block);
5636 17 : add_phi_arg (new_phi, orig_reduc_input, single_succ_edge (exit_bb),
5637 : UNKNOWN_LOCATION);
5638 17 : add_phi_arg (new_phi,
5639 17 : VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)->reduc_input,
5640 : loop_vinfo->skip_this_loop_edge, UNKNOWN_LOCATION);
5641 17 : exit_gsi = gsi_after_labels (reduc_block);
5642 : }
5643 :
5644 : /* Shouldn't be used beyond this point. */
5645 22064 : exit_bb = nullptr;
5646 :
5647 : /* If we are operating on a mask vector and do not support direct mask
5648 : reduction, work on a bool data vector instead of a mask vector. */
5649 22064 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
5650 235 : && VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info)
5651 22249 : && vectype != VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info))
5652 : {
5653 185 : compute_vectype = vectype = VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info);
5654 185 : gimple_seq stmts = NULL;
5655 378 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
5656 386 : reduc_inputs[i] = gimple_build (&stmts, VEC_COND_EXPR, vectype,
5657 193 : reduc_inputs[i],
5658 : build_one_cst (vectype),
5659 : build_zero_cst (vectype));
5660 185 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5661 : }
5662 :
5663 22064 : if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5664 99 : && reduc_fn != IFN_LAST)
5665 : {
5666 : /* For condition reductions, we have a vector (REDUC_INPUTS 0) containing
5667 : various data values where the condition matched and another vector
5668 : (INDUCTION_INDEX) containing all the indexes of those matches. We
5669 : need to extract the last matching index (which will be the index with
5670 : highest value) and use this to index into the data vector.
5671 : For the case where there were no matches, the data vector will contain
5672 : all default values and the index vector will be all zeros. */
5673 :
5674 : /* Get various versions of the type of the vector of indexes. */
5675 14 : tree index_vec_type = TREE_TYPE (induction_index);
5676 14 : gcc_checking_assert (TYPE_UNSIGNED (index_vec_type));
5677 14 : tree index_scalar_type = TREE_TYPE (index_vec_type);
5678 14 : tree index_vec_cmp_type = truth_type_for (index_vec_type);
5679 :
5680 : /* Get an unsigned integer version of the type of the data vector. */
5681 14 : int scalar_precision
5682 14 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type));
5683 14 : tree scalar_type_unsigned = make_unsigned_type (scalar_precision);
5684 14 : tree vectype_unsigned = get_same_sized_vectype (scalar_type_unsigned,
5685 : vectype);
5686 :
5687 : /* First we need to create a vector (ZERO_VEC) of zeros and another
5688 : vector (MAX_INDEX_VEC) filled with the last matching index, which we
5689 : can create using a MAX reduction and then expanding.
5690 : In the case where the loop never made any matches, the max index will
5691 : be zero. */
5692 :
5693 : /* Vector of {0, 0, 0,...}. */
5694 14 : tree zero_vec = build_zero_cst (vectype);
5695 :
5696 : /* Find maximum value from the vector of found indexes. */
5697 14 : tree max_index = make_ssa_name (index_scalar_type);
5698 14 : gcall *max_index_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5699 : 1, induction_index);
5700 14 : gimple_call_set_lhs (max_index_stmt, max_index);
5701 14 : gsi_insert_before (&exit_gsi, max_index_stmt, GSI_SAME_STMT);
5702 :
5703 : /* Vector of {max_index, max_index, max_index,...}. */
5704 14 : tree max_index_vec = make_ssa_name (index_vec_type);
5705 14 : tree max_index_vec_rhs = build_vector_from_val (index_vec_type,
5706 : max_index);
5707 14 : gimple *max_index_vec_stmt = gimple_build_assign (max_index_vec,
5708 : max_index_vec_rhs);
5709 14 : gsi_insert_before (&exit_gsi, max_index_vec_stmt, GSI_SAME_STMT);
5710 :
5711 : /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
5712 : with the vector (INDUCTION_INDEX) of found indexes, choosing values
5713 : from the data vector (REDUC_INPUTS 0) for matches, 0 (ZERO_VEC)
5714 : otherwise. Only one value should match, resulting in a vector
5715 : (VEC_COND) with one data value and the rest zeros.
5716 : In the case where the loop never made any matches, every index will
5717 : match, resulting in a vector with all data values (which will all be
5718 : the default value). */
5719 :
5720 : /* Compare the max index vector to the vector of found indexes to find
5721 : the position of the max value. */
5722 14 : tree vec_compare = make_ssa_name (index_vec_cmp_type);
5723 14 : gimple *vec_compare_stmt = gimple_build_assign (vec_compare, EQ_EXPR,
5724 : induction_index,
5725 : max_index_vec);
5726 14 : gsi_insert_before (&exit_gsi, vec_compare_stmt, GSI_SAME_STMT);
5727 :
5728 : /* Use the compare to choose either values from the data vector or
5729 : zero. */
5730 14 : tree vec_cond = make_ssa_name (vectype);
5731 14 : gimple *vec_cond_stmt = gimple_build_assign (vec_cond, VEC_COND_EXPR,
5732 : vec_compare,
5733 14 : reduc_inputs[0],
5734 : zero_vec);
5735 14 : gsi_insert_before (&exit_gsi, vec_cond_stmt, GSI_SAME_STMT);
5736 :
5737 : /* Finally we need to extract the data value from the vector (VEC_COND)
5738 : into a scalar (MATCHED_DATA_REDUC). Logically we want to do a OR
5739 : reduction, but because this doesn't exist, we can use a MAX reduction
5740 : instead. The data value might be signed or a float so we need to cast
5741 : it first.
5742 : In the case where the loop never made any matches, the data values are
5743 : all identical, and so will reduce down correctly. */
5744 :
5745 : /* Make the matched data values unsigned. */
5746 14 : tree vec_cond_cast = make_ssa_name (vectype_unsigned);
5747 14 : tree vec_cond_cast_rhs = build1 (VIEW_CONVERT_EXPR, vectype_unsigned,
5748 : vec_cond);
5749 14 : gimple *vec_cond_cast_stmt = gimple_build_assign (vec_cond_cast,
5750 : VIEW_CONVERT_EXPR,
5751 : vec_cond_cast_rhs);
5752 14 : gsi_insert_before (&exit_gsi, vec_cond_cast_stmt, GSI_SAME_STMT);
5753 :
5754 : /* Reduce down to a scalar value. */
5755 14 : tree data_reduc = make_ssa_name (scalar_type_unsigned);
5756 14 : gcall *data_reduc_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
5757 : 1, vec_cond_cast);
5758 14 : gimple_call_set_lhs (data_reduc_stmt, data_reduc);
5759 14 : gsi_insert_before (&exit_gsi, data_reduc_stmt, GSI_SAME_STMT);
5760 :
5761 : /* Convert the reduced value back to the result type and set as the
5762 : result. */
5763 14 : gimple_seq stmts = NULL;
5764 14 : new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR, scalar_type,
5765 : data_reduc);
5766 14 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5767 14 : scalar_results.safe_push (new_temp);
5768 14 : }
5769 22050 : else if (VECT_REDUC_INFO_TYPE (reduc_info) == COND_REDUCTION
5770 85 : && reduc_fn == IFN_LAST)
5771 : {
5772 : /* Condition reduction without supported IFN_REDUC_MAX. Generate
5773 : idx = 0;
5774 : idx_val = induction_index[0];
5775 : val = data_reduc[0];
5776 : for (idx = 0, val = init, i = 0; i < nelts; ++i)
5777 : if (induction_index[i] > idx_val)
5778 : val = data_reduc[i], idx_val = induction_index[i];
5779 : return val; */
5780 :
5781 85 : tree data_eltype = TREE_TYPE (vectype);
5782 85 : tree idx_eltype = TREE_TYPE (TREE_TYPE (induction_index));
5783 85 : unsigned HOST_WIDE_INT el_size = tree_to_uhwi (TYPE_SIZE (idx_eltype));
5784 85 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index));
5785 : /* Enforced by vectorizable_reduction, which ensures we have target
5786 : support before allowing a conditional reduction on variable-length
5787 : vectors. */
5788 85 : unsigned HOST_WIDE_INT v_size = el_size * nunits.to_constant ();
5789 85 : tree idx_val = NULL_TREE, val = NULL_TREE;
5790 511 : for (unsigned HOST_WIDE_INT off = 0; off < v_size; off += el_size)
5791 : {
5792 426 : tree old_idx_val = idx_val;
5793 426 : tree old_val = val;
5794 426 : idx_val = make_ssa_name (idx_eltype);
5795 426 : epilog_stmt = gimple_build_assign (idx_val, BIT_FIELD_REF,
5796 : build3 (BIT_FIELD_REF, idx_eltype,
5797 : induction_index,
5798 426 : bitsize_int (el_size),
5799 426 : bitsize_int (off)));
5800 426 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5801 426 : val = make_ssa_name (data_eltype);
5802 852 : epilog_stmt = gimple_build_assign (val, BIT_FIELD_REF,
5803 : build3 (BIT_FIELD_REF,
5804 : data_eltype,
5805 426 : reduc_inputs[0],
5806 426 : bitsize_int (el_size),
5807 426 : bitsize_int (off)));
5808 426 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5809 426 : if (off != 0)
5810 : {
5811 341 : tree new_idx_val = idx_val;
5812 341 : if (off != v_size - el_size)
5813 : {
5814 256 : new_idx_val = make_ssa_name (idx_eltype);
5815 256 : epilog_stmt = gimple_build_assign (new_idx_val,
5816 : MAX_EXPR, idx_val,
5817 : old_idx_val);
5818 256 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5819 : }
5820 341 : tree cond = make_ssa_name (boolean_type_node);
5821 341 : epilog_stmt = gimple_build_assign (cond, GT_EXPR,
5822 : idx_val, old_idx_val);
5823 341 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5824 341 : tree new_val = make_ssa_name (data_eltype);
5825 341 : epilog_stmt = gimple_build_assign (new_val, COND_EXPR,
5826 : cond, val, old_val);
5827 341 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5828 341 : idx_val = new_idx_val;
5829 341 : val = new_val;
5830 : }
5831 : }
5832 : /* Convert the reduced value back to the result type and set as the
5833 : result. */
5834 85 : gimple_seq stmts = NULL;
5835 85 : val = gimple_convert (&stmts, scalar_type, val);
5836 85 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5837 85 : scalar_results.safe_push (val);
5838 85 : }
5839 :
5840 : /* 2.3 Create the reduction code, using one of the three schemes described
5841 : above. In SLP we simply need to extract all the elements from the
5842 : vector (without reducing them), so we use scalar shifts. */
5843 21965 : else if (reduc_fn != IFN_LAST && (!slp_reduc || group_size == 1))
5844 : {
5845 20147 : tree tmp;
5846 20147 : tree vec_elem_type;
5847 :
5848 : /* Case 1: Create:
5849 : v_out2 = reduc_expr <v_out1> */
5850 :
5851 20147 : if (dump_enabled_p ())
5852 1529 : dump_printf_loc (MSG_NOTE, vect_location,
5853 : "Reduce using direct vector reduction.\n");
5854 :
5855 20147 : gimple_seq stmts = NULL;
5856 20147 : vec_elem_type = TREE_TYPE (vectype);
5857 20147 : new_temp = gimple_build (&stmts, as_combined_fn (reduc_fn),
5858 20147 : vec_elem_type, reduc_inputs[0]);
5859 20147 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
5860 20147 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5861 :
5862 20147 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
5863 62 : && induc_val)
5864 : {
5865 : /* Earlier we set the initial value to be a vector if induc_val
5866 : values. Check the result and if it is induc_val then replace
5867 : with the original initial value, unless induc_val is
5868 : the same as initial_def already. */
5869 60 : tree zcompare = make_ssa_name (boolean_type_node);
5870 60 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
5871 : new_temp, induc_val);
5872 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5873 60 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
5874 60 : tmp = make_ssa_name (new_scalar_dest);
5875 60 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
5876 : initial_def, new_temp);
5877 60 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
5878 60 : new_temp = tmp;
5879 : }
5880 :
5881 20147 : scalar_results.safe_push (new_temp);
5882 20147 : }
5883 1623 : else if (direct_slp_reduc)
5884 : {
5885 : /* Here we create one vector for each of the GROUP_SIZE results,
5886 : with the elements for other SLP statements replaced with the
5887 : neutral value. We can then do a normal reduction on each vector. */
5888 :
5889 : /* Enforced by vectorizable_reduction. */
5890 : gcc_assert (reduc_inputs.length () == 1);
5891 : gcc_assert (pow2p_hwi (group_size));
5892 :
5893 : gimple_seq seq = NULL;
5894 :
5895 : /* Build a vector {0, 1, 2, ...}, with the same number of elements
5896 : and the same element size as VECTYPE. */
5897 : tree index = build_index_vector (vectype, 0, 1);
5898 : tree index_type = TREE_TYPE (index);
5899 : tree index_elt_type = TREE_TYPE (index_type);
5900 : tree mask_type = truth_type_for (index_type);
5901 :
5902 : /* Create a vector that, for each element, identifies which of
5903 : the results should use it. */
5904 : tree index_mask = build_int_cst (index_elt_type, group_size - 1);
5905 : index = gimple_build (&seq, BIT_AND_EXPR, index_type, index,
5906 : build_vector_from_val (index_type, index_mask));
5907 :
5908 : /* Get a neutral vector value. This is simply a splat of the neutral
5909 : scalar value if we have one, otherwise the initial scalar value
5910 : is itself a neutral value. */
5911 : tree vector_identity = NULL_TREE;
5912 : tree neutral_op = neutral_op_for_reduction (TREE_TYPE (vectype), code,
5913 : NULL_TREE, false);
5914 : if (neutral_op)
5915 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
5916 : neutral_op);
5917 : for (unsigned int i = 0; i < group_size; ++i)
5918 : {
5919 : /* If there's no universal neutral value, we can use the
5920 : initial scalar value from the original PHI. This is used
5921 : for MIN and MAX reduction, for example. */
5922 : if (!neutral_op)
5923 : {
5924 : tree scalar_value
5925 : = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[i];
5926 : scalar_value = gimple_convert (&seq, TREE_TYPE (vectype),
5927 : scalar_value);
5928 : vector_identity = gimple_build_vector_from_val (&seq, vectype,
5929 : scalar_value);
5930 : }
5931 :
5932 : /* Calculate the equivalent of:
5933 :
5934 : sel[j] = (index[j] == i);
5935 :
5936 : which selects the elements of REDUC_INPUTS[0] that should
5937 : be included in the result. */
5938 : tree compare_val = build_int_cst (index_elt_type, i);
5939 : compare_val = build_vector_from_val (index_type, compare_val);
5940 : tree sel = gimple_build (&seq, EQ_EXPR, mask_type,
5941 : index, compare_val);
5942 :
5943 : /* Calculate the equivalent of:
5944 :
5945 : vec = seq ? reduc_inputs[0] : vector_identity;
5946 :
5947 : VEC is now suitable for a full vector reduction. */
5948 : tree vec = gimple_build (&seq, VEC_COND_EXPR, vectype,
5949 : sel, reduc_inputs[0], vector_identity);
5950 :
5951 : /* Do the reduction and convert it to the appropriate type. */
5952 : tree scalar = gimple_build (&seq, as_combined_fn (reduc_fn),
5953 : TREE_TYPE (vectype), vec);
5954 : scalar = gimple_convert (&seq, scalar_type, scalar);
5955 : scalar_results.safe_push (scalar);
5956 : }
5957 : gsi_insert_seq_before (&exit_gsi, seq, GSI_SAME_STMT);
5958 : }
5959 : else
5960 : {
5961 1623 : bool reduce_with_shift;
5962 1623 : tree vec_temp;
5963 :
5964 1623 : gcc_assert (slp_reduc || reduc_inputs.length () == 1);
5965 :
5966 : /* See if the target wants to do the final (shift) reduction
5967 : in a vector mode of smaller size and first reduce upper/lower
5968 : halves against each other. */
5969 1818 : enum machine_mode mode1 = mode;
5970 1818 : tree stype = TREE_TYPE (vectype);
5971 1818 : if (compute_vectype != vectype)
5972 : {
5973 461 : stype = unsigned_type_for (stype);
5974 461 : gimple_seq stmts = NULL;
5975 980 : for (unsigned i = 0; i < reduc_inputs.length (); ++i)
5976 : {
5977 519 : tree new_temp = gimple_build (&stmts, VIEW_CONVERT_EXPR,
5978 519 : compute_vectype, reduc_inputs[i]);
5979 519 : reduc_inputs[i] = new_temp;
5980 : }
5981 461 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
5982 : }
5983 1818 : unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
5984 1818 : unsigned nunits1 = nunits;
5985 1818 : if ((mode1 = targetm.vectorize.split_reduction (mode)) != mode
5986 1818 : && reduc_inputs.length () == 1)
5987 : {
5988 41 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
5989 : /* For SLP reductions we have to make sure lanes match up, but
5990 : since we're doing individual element final reduction reducing
5991 : vector width here is even more important.
5992 : ??? We can also separate lanes with permutes, for the common
5993 : case of power-of-two group-size odd/even extracts would work. */
5994 41 : if (slp_reduc && nunits != nunits1)
5995 : {
5996 41 : nunits1 = least_common_multiple (nunits1, group_size);
5997 82 : gcc_assert (exact_log2 (nunits1) != -1 && nunits1 <= nunits);
5998 : }
5999 : }
6000 1777 : else if (!slp_reduc
6001 1777 : && (mode1 = targetm.vectorize.split_reduction (mode)) != mode)
6002 0 : nunits1 = GET_MODE_NUNITS (mode1).to_constant ();
6003 :
6004 1818 : tree vectype1 = compute_vectype;
6005 1818 : if (mode1 != mode)
6006 : {
6007 47 : vectype1 = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
6008 47 : stype, nunits1);
6009 : /* First reduce the vector to the desired vector size we should
6010 : do shift reduction on by combining upper and lower halves. */
6011 47 : gimple_seq stmts = NULL;
6012 47 : new_temp = vect_create_partial_epilog (reduc_inputs[0], vectype1,
6013 : code, &stmts);
6014 47 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6015 47 : reduc_inputs[0] = new_temp;
6016 : }
6017 :
6018 1818 : reduce_with_shift = have_whole_vector_shift (mode1);
6019 749 : if (!VECTOR_MODE_P (mode1)
6020 2565 : || !directly_supported_p (code, vectype1))
6021 : reduce_with_shift = false;
6022 :
6023 1801 : if (reduce_with_shift && (!slp_reduc || group_size == 1))
6024 : {
6025 1557 : int element_bitsize = vector_element_bits (vectype1);
6026 : /* Enforced by vectorizable_reduction, which disallows SLP reductions
6027 : for variable-length vectors and also requires direct target support
6028 : for loop reductions. */
6029 1557 : int nelements = TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6030 1557 : vec_perm_builder sel;
6031 1557 : vec_perm_indices indices;
6032 :
6033 1557 : int elt_offset;
6034 :
6035 1557 : tree zero_vec = build_zero_cst (vectype1);
6036 : /* Case 2: Create:
6037 : for (offset = nelements/2; offset >= 1; offset/=2)
6038 : {
6039 : Create: va' = vec_shift <va, offset>
6040 : Create: va = vop <va, va'>
6041 : } */
6042 :
6043 1557 : if (dump_enabled_p ())
6044 378 : dump_printf_loc (MSG_NOTE, vect_location,
6045 : "Reduce using vector shifts\n");
6046 :
6047 1557 : gimple_seq stmts = NULL;
6048 1557 : new_temp = gimple_convert (&stmts, vectype1, reduc_inputs[0]);
6049 1557 : for (elt_offset = nelements / 2;
6050 3421 : elt_offset >= 1;
6051 1864 : elt_offset /= 2)
6052 : {
6053 1864 : calc_vec_perm_mask_for_shift (elt_offset, nelements, &sel);
6054 1864 : indices.new_vector (sel, 2, nelements);
6055 1864 : tree mask = vect_gen_perm_mask_any (vectype1, indices);
6056 1864 : new_name = gimple_build (&stmts, VEC_PERM_EXPR, vectype1,
6057 : new_temp, zero_vec, mask);
6058 1864 : new_temp = gimple_build (&stmts, code,
6059 : vectype1, new_name, new_temp);
6060 : }
6061 :
6062 : /* 2.4 Extract the final scalar result. Create:
6063 : s_out3 = extract_field <v_out2, bitpos> */
6064 :
6065 1557 : if (dump_enabled_p ())
6066 378 : dump_printf_loc (MSG_NOTE, vect_location,
6067 : "extract scalar result\n");
6068 :
6069 1557 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype1),
6070 1557 : new_temp, bitsize_int (element_bitsize),
6071 1557 : bitsize_zero_node);
6072 1557 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6073 1557 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6074 1557 : scalar_results.safe_push (new_temp);
6075 1557 : }
6076 : else
6077 : {
6078 : /* Case 3: Create:
6079 : s = extract_field <v_out2, 0>
6080 : for (offset = element_size;
6081 : offset < vector_size;
6082 : offset += element_size;)
6083 : {
6084 : Create: s' = extract_field <v_out2, offset>
6085 : Create: s = op <s, s'> // For non SLP cases
6086 : } */
6087 :
6088 261 : if (dump_enabled_p ())
6089 143 : dump_printf_loc (MSG_NOTE, vect_location,
6090 : "Reduce using scalar code.\n");
6091 :
6092 261 : tree compute_type = TREE_TYPE (vectype1);
6093 261 : unsigned element_bitsize = vector_element_bits (vectype1);
6094 261 : unsigned vec_size_in_bits = element_bitsize
6095 261 : * TYPE_VECTOR_SUBPARTS (vectype1).to_constant ();
6096 261 : tree bitsize = bitsize_int (element_bitsize);
6097 261 : gimple_seq stmts = NULL;
6098 677 : FOR_EACH_VEC_ELT (reduc_inputs, i, vec_temp)
6099 : {
6100 416 : unsigned bit_offset;
6101 832 : new_temp = gimple_build (&stmts, BIT_FIELD_REF, compute_type,
6102 416 : vec_temp, bitsize, bitsize_zero_node);
6103 :
6104 : /* In SLP we don't need to apply reduction operation, so we just
6105 : collect s' values in SCALAR_RESULTS. */
6106 416 : if (slp_reduc)
6107 406 : scalar_results.safe_push (new_temp);
6108 :
6109 416 : for (bit_offset = element_bitsize;
6110 1402 : bit_offset < vec_size_in_bits;
6111 986 : bit_offset += element_bitsize)
6112 : {
6113 986 : tree bitpos = bitsize_int (bit_offset);
6114 986 : new_name = gimple_build (&stmts, BIT_FIELD_REF,
6115 : compute_type, vec_temp,
6116 : bitsize, bitpos);
6117 986 : if (slp_reduc)
6118 : {
6119 : /* In SLP we don't need to apply reduction operation, so
6120 : we just collect s' values in SCALAR_RESULTS. */
6121 976 : new_temp = new_name;
6122 976 : scalar_results.safe_push (new_name);
6123 : }
6124 : else
6125 10 : new_temp = gimple_build (&stmts, code, compute_type,
6126 : new_name, new_temp);
6127 : }
6128 : }
6129 :
6130 : /* The only case where we need to reduce scalar results in a SLP
6131 : reduction, is unrolling. If the size of SCALAR_RESULTS is
6132 : greater than GROUP_SIZE, we reduce them combining elements modulo
6133 : GROUP_SIZE. */
6134 261 : if (slp_reduc)
6135 : {
6136 251 : tree res, first_res, new_res;
6137 :
6138 : /* Reduce multiple scalar results in case of SLP unrolling. */
6139 907 : for (j = group_size; scalar_results.iterate (j, &res);
6140 : j++)
6141 : {
6142 656 : first_res = scalar_results[j % group_size];
6143 656 : new_res = gimple_build (&stmts, code, compute_type,
6144 : first_res, res);
6145 656 : scalar_results[j % group_size] = new_res;
6146 : }
6147 251 : scalar_results.truncate (group_size);
6148 1228 : for (k = 0; k < group_size; k++)
6149 1452 : scalar_results[k] = gimple_convert (&stmts, scalar_type,
6150 726 : scalar_results[k]);
6151 : }
6152 : else
6153 : {
6154 : /* Reduction chain - we have one scalar to keep in
6155 : SCALAR_RESULTS. */
6156 10 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6157 10 : scalar_results.safe_push (new_temp);
6158 : }
6159 :
6160 261 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6161 : }
6162 :
6163 1818 : if ((VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
6164 0 : && induc_val)
6165 : {
6166 : /* Earlier we set the initial value to be a vector if induc_val
6167 : values. Check the result and if it is induc_val then replace
6168 : with the original initial value, unless induc_val is
6169 : the same as initial_def already. */
6170 0 : tree zcompare = make_ssa_name (boolean_type_node);
6171 0 : epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
6172 0 : scalar_results[0], induc_val);
6173 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6174 0 : tree initial_def = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info)[0];
6175 0 : tree tmp = make_ssa_name (new_scalar_dest);
6176 0 : epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
6177 0 : initial_def, scalar_results[0]);
6178 0 : gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
6179 0 : scalar_results[0] = tmp;
6180 : }
6181 : }
6182 :
6183 : /* 2.5 Adjust the final result by the initial value of the reduction
6184 : variable. (When such adjustment is not needed, then
6185 : 'adjustment_def' is zero). For example, if code is PLUS we create:
6186 : new_temp = loop_exit_def + adjustment_def */
6187 :
6188 22064 : if (adjustment_def)
6189 : {
6190 15568 : gcc_assert (!slp_reduc || group_size == 1);
6191 15568 : gimple_seq stmts = NULL;
6192 15568 : if (double_reduc)
6193 : {
6194 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def)));
6195 0 : adjustment_def = gimple_convert (&stmts, vectype, adjustment_def);
6196 0 : new_temp = gimple_build (&stmts, code, vectype,
6197 0 : reduc_inputs[0], adjustment_def);
6198 : }
6199 : else
6200 : {
6201 15568 : new_temp = scalar_results[0];
6202 15568 : gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
6203 15568 : adjustment_def = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6204 : adjustment_def);
6205 15568 : new_temp = gimple_convert (&stmts, TREE_TYPE (compute_vectype),
6206 : new_temp);
6207 15568 : new_temp = gimple_build (&stmts, code, TREE_TYPE (compute_vectype),
6208 : new_temp, adjustment_def);
6209 15568 : new_temp = gimple_convert (&stmts, scalar_type, new_temp);
6210 : }
6211 :
6212 15568 : epilog_stmt = gimple_seq_last_stmt (stmts);
6213 15568 : gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
6214 15568 : scalar_results[0] = new_temp;
6215 : }
6216 :
6217 : /* Record this operation if it could be reused by the epilogue loop. */
6218 22064 : if (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION
6219 22064 : && reduc_inputs.length () == 1)
6220 21850 : loop_vinfo->reusable_accumulators.put (scalar_results[0],
6221 : { orig_reduc_input, reduc_info });
6222 :
6223 : /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
6224 : phis with new adjusted scalar results, i.e., replace use <s_out0>
6225 : with use <s_out4>.
6226 :
6227 : Transform:
6228 : loop_exit:
6229 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6230 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6231 : v_out2 = reduce <v_out1>
6232 : s_out3 = extract_field <v_out2, 0>
6233 : s_out4 = adjust_result <s_out3>
6234 : use <s_out0>
6235 : use <s_out0>
6236 :
6237 : into:
6238 :
6239 : loop_exit:
6240 : s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6241 : v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6242 : v_out2 = reduce <v_out1>
6243 : s_out3 = extract_field <v_out2, 0>
6244 : s_out4 = adjust_result <s_out3>
6245 : use <s_out4>
6246 : use <s_out4> */
6247 :
6248 44128 : gcc_assert (live_out_stmts.size () == scalar_results.length ());
6249 22064 : auto_vec<gimple *> phis;
6250 44603 : for (k = 0; k < live_out_stmts.size (); k++)
6251 : {
6252 22539 : stmt_vec_info scalar_stmt_info = vect_orig_stmt (live_out_stmts[k]);
6253 22539 : tree scalar_dest = gimple_get_lhs (scalar_stmt_info->stmt);
6254 :
6255 : /* Find the loop-closed-use at the loop exit of the original scalar
6256 : result. (The reduction result is expected to have two immediate uses,
6257 : one at the latch block, and one at the loop exit). Note with
6258 : early break we can have two exit blocks, so pick the correct PHI. */
6259 91418 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
6260 68879 : if (!is_gimple_debug (USE_STMT (use_p))
6261 68879 : && !flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
6262 : {
6263 22532 : gcc_assert (is_a <gphi *> (USE_STMT (use_p)));
6264 22532 : if (gimple_bb (USE_STMT (use_p)) == loop_exit->dest)
6265 22524 : phis.safe_push (USE_STMT (use_p));
6266 22539 : }
6267 :
6268 45063 : FOR_EACH_VEC_ELT (phis, i, exit_phi)
6269 : {
6270 : /* Replace the uses: */
6271 22524 : orig_name = PHI_RESULT (exit_phi);
6272 :
6273 : /* Look for a single use at the target of the skip edge. */
6274 22524 : if (unify_with_main_loop_p)
6275 : {
6276 30 : use_operand_p use_p;
6277 30 : gimple *user;
6278 30 : if (!single_imm_use (orig_name, &use_p, &user))
6279 0 : gcc_unreachable ();
6280 30 : orig_name = gimple_get_lhs (user);
6281 : }
6282 :
6283 22524 : scalar_result = scalar_results[k];
6284 60792 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
6285 : {
6286 38268 : gphi *use_phi = dyn_cast <gphi *> (use_stmt);
6287 76580 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6288 : {
6289 38290 : if (use_phi
6290 38290 : && (phi_arg_edge_from_use (use_p)->flags & EDGE_ABNORMAL))
6291 : {
6292 0 : gcc_assert (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (orig_name));
6293 0 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (scalar_result) = 1;
6294 : }
6295 38290 : SET_USE (use_p, scalar_result);
6296 : }
6297 38268 : update_stmt (use_stmt);
6298 22524 : }
6299 : }
6300 :
6301 22539 : phis.truncate (0);
6302 : }
6303 22064 : }
6304 :
6305 : /* Return a vector of type VECTYPE that is equal to the vector select
6306 : operation "MASK ? VEC : IDENTITY". Insert the select statements
6307 : before GSI. */
6308 :
6309 : static tree
6310 9 : merge_with_identity (gimple_stmt_iterator *gsi, tree mask, tree vectype,
6311 : tree vec, tree identity)
6312 : {
6313 9 : tree cond = make_temp_ssa_name (vectype, NULL, "cond");
6314 9 : gimple *new_stmt = gimple_build_assign (cond, VEC_COND_EXPR,
6315 : mask, vec, identity);
6316 9 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6317 9 : return cond;
6318 : }
6319 :
6320 : /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
6321 : order, starting with LHS. Insert the extraction statements before GSI and
6322 : associate the new scalar SSA names with variable SCALAR_DEST.
6323 : If MASK is nonzero mask the input and then operate on it unconditionally.
6324 : Return the SSA name for the result. */
6325 :
6326 : static tree
6327 1262 : vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
6328 : tree_code code, tree lhs, tree vector_rhs,
6329 : tree mask)
6330 : {
6331 1262 : tree vectype = TREE_TYPE (vector_rhs);
6332 1262 : tree scalar_type = TREE_TYPE (vectype);
6333 1262 : tree bitsize = TYPE_SIZE (scalar_type);
6334 1262 : unsigned HOST_WIDE_INT vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
6335 1262 : unsigned HOST_WIDE_INT element_bitsize = tree_to_uhwi (bitsize);
6336 :
6337 : /* Re-create a VEC_COND_EXPR to mask the input here in order to be able
6338 : to perform an unconditional element-wise reduction of it. */
6339 1262 : if (mask)
6340 : {
6341 76 : tree masked_vector_rhs = make_temp_ssa_name (vectype, NULL,
6342 : "masked_vector_rhs");
6343 76 : tree neutral_op = neutral_op_for_reduction (scalar_type, code, NULL_TREE,
6344 : false);
6345 76 : tree vector_identity = build_vector_from_val (vectype, neutral_op);
6346 76 : gassign *select = gimple_build_assign (masked_vector_rhs, VEC_COND_EXPR,
6347 : mask, vector_rhs, vector_identity);
6348 76 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6349 76 : vector_rhs = masked_vector_rhs;
6350 : }
6351 :
6352 1262 : for (unsigned HOST_WIDE_INT bit_offset = 0;
6353 5200 : bit_offset < vec_size_in_bits;
6354 3938 : bit_offset += element_bitsize)
6355 : {
6356 3938 : tree bitpos = bitsize_int (bit_offset);
6357 3938 : tree rhs = build3 (BIT_FIELD_REF, scalar_type, vector_rhs,
6358 : bitsize, bitpos);
6359 :
6360 3938 : gassign *stmt = gimple_build_assign (scalar_dest, rhs);
6361 3938 : rhs = make_ssa_name (scalar_dest, stmt);
6362 3938 : gimple_assign_set_lhs (stmt, rhs);
6363 3938 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6364 : /* Fold the vector extract, combining it with a previous reversal
6365 : like seen in PR90579. */
6366 3938 : auto gsi2 = gsi_for_stmt (stmt);
6367 3938 : if (fold_stmt (&gsi2, follow_all_ssa_edges))
6368 450 : update_stmt (gsi_stmt (gsi2));
6369 :
6370 3938 : stmt = gimple_build_assign (scalar_dest, code, lhs, rhs);
6371 3938 : tree new_name = make_ssa_name (scalar_dest, stmt);
6372 3938 : gimple_assign_set_lhs (stmt, new_name);
6373 3938 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6374 3938 : lhs = new_name;
6375 : }
6376 1262 : return lhs;
6377 : }
6378 :
6379 : /* Get a masked internal function equivalent to REDUC_FN. VECTYPE_IN is the
6380 : type of the vector input. */
6381 :
6382 : static internal_fn
6383 3080 : get_masked_reduction_fn (internal_fn reduc_fn, tree vectype_in)
6384 : {
6385 3080 : internal_fn mask_reduc_fn;
6386 3080 : internal_fn mask_len_reduc_fn;
6387 :
6388 3080 : switch (reduc_fn)
6389 : {
6390 0 : case IFN_FOLD_LEFT_PLUS:
6391 0 : mask_reduc_fn = IFN_MASK_FOLD_LEFT_PLUS;
6392 0 : mask_len_reduc_fn = IFN_MASK_LEN_FOLD_LEFT_PLUS;
6393 0 : break;
6394 :
6395 : default:
6396 : return IFN_LAST;
6397 : }
6398 :
6399 0 : if (direct_internal_fn_supported_p (mask_reduc_fn, vectype_in,
6400 : OPTIMIZE_FOR_SPEED))
6401 : return mask_reduc_fn;
6402 0 : if (direct_internal_fn_supported_p (mask_len_reduc_fn, vectype_in,
6403 : OPTIMIZE_FOR_SPEED))
6404 0 : return mask_len_reduc_fn;
6405 : return IFN_LAST;
6406 : }
6407 :
6408 : /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
6409 : statement that sets the live-out value. REDUC_DEF_STMT is the phi
6410 : statement. CODE is the operation performed by STMT_INFO and OPS are
6411 : its scalar operands. REDUC_INDEX is the index of the operand in
6412 : OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
6413 : implements in-order reduction, or IFN_LAST if we should open-code it.
6414 : VECTYPE_IN is the type of the vector input. MASKS specifies the masks
6415 : that should be used to control the operation in a fully-masked loop. */
6416 :
6417 : static bool
6418 930 : vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
6419 : stmt_vec_info stmt_info,
6420 : gimple_stmt_iterator *gsi,
6421 : slp_tree slp_node,
6422 : code_helper code, internal_fn reduc_fn,
6423 : int num_ops, tree vectype_in,
6424 : int reduc_index, vec_loop_masks *masks,
6425 : vec_loop_lens *lens)
6426 : {
6427 930 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
6428 930 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
6429 930 : internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
6430 :
6431 930 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
6432 :
6433 930 : bool is_cond_op = false;
6434 930 : if (!code.is_tree_code ())
6435 : {
6436 29 : code = conditional_internal_fn_code (internal_fn (code));
6437 29 : gcc_assert (code != ERROR_MARK);
6438 : is_cond_op = true;
6439 : }
6440 :
6441 930 : gcc_assert (TREE_CODE_LENGTH (tree_code (code)) == binary_op);
6442 :
6443 930 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out),
6444 : TYPE_VECTOR_SUBPARTS (vectype_in)));
6445 :
6446 : /* ??? We should, when transforming the cycle PHI, record the existing
6447 : scalar def as vector def so looking up the vector def works. This
6448 : would also allow generalizing this for reduction paths of length > 1
6449 : and/or SLP reductions. */
6450 930 : slp_tree reduc_node = SLP_TREE_CHILDREN (slp_node)[reduc_index];
6451 930 : stmt_vec_info reduc_var_def = SLP_TREE_SCALAR_STMTS (reduc_node)[0];
6452 930 : tree reduc_var = gimple_get_lhs (STMT_VINFO_STMT (reduc_var_def));
6453 :
6454 : /* The operands either come from a binary operation or an IFN_COND operation.
6455 : The former is a gimple assign with binary rhs and the latter is a
6456 : gimple call with four arguments. */
6457 930 : gcc_assert (num_ops == 2 || num_ops == 4);
6458 :
6459 930 : auto_vec<tree> vec_oprnds0, vec_opmask;
6460 930 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[(is_cond_op ? 2 : 0)
6461 930 : + (1 - reduc_index)],
6462 : &vec_oprnds0);
6463 : /* For an IFN_COND_OP we also need the vector mask operand. */
6464 930 : if (is_cond_op)
6465 29 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], &vec_opmask);
6466 :
6467 : /* The transform below relies on preserving the original scalar PHI
6468 : and its latch def which we replace. So work backwards from there. */
6469 930 : tree scalar_dest
6470 930 : = gimple_phi_arg_def_from_edge (as_a <gphi *> (STMT_VINFO_STMT
6471 : (reduc_var_def)),
6472 930 : loop_latch_edge (loop));
6473 930 : stmt_vec_info scalar_dest_def_info
6474 930 : = vect_stmt_to_vectorize (loop_vinfo->lookup_def (scalar_dest));
6475 930 : tree scalar_type = TREE_TYPE (scalar_dest);
6476 :
6477 930 : int vec_num = vec_oprnds0.length ();
6478 930 : tree vec_elem_type = TREE_TYPE (vectype_out);
6479 930 : gcc_checking_assert (useless_type_conversion_p (scalar_type, vec_elem_type));
6480 :
6481 930 : tree vector_identity = NULL_TREE;
6482 930 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6483 : {
6484 2 : vector_identity = build_zero_cst (vectype_out);
6485 2 : if (!HONOR_SIGNED_ZEROS (vectype_out))
6486 : ;
6487 : else
6488 : {
6489 2 : gcc_assert (!HONOR_SIGN_DEPENDENT_ROUNDING (vectype_out));
6490 2 : vector_identity = const_unop (NEGATE_EXPR, vectype_out,
6491 : vector_identity);
6492 : }
6493 : }
6494 :
6495 930 : tree scalar_dest_var = vect_create_destination_var (scalar_dest, NULL);
6496 930 : int i;
6497 930 : tree def0;
6498 3122 : FOR_EACH_VEC_ELT (vec_oprnds0, i, def0)
6499 : {
6500 1262 : gimple *new_stmt;
6501 1262 : tree mask = NULL_TREE;
6502 1262 : tree len = NULL_TREE;
6503 1262 : tree bias = NULL_TREE;
6504 1262 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
6505 : {
6506 9 : tree loop_mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
6507 : vec_num, vectype_in, i);
6508 9 : if (is_cond_op)
6509 9 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (loop_mask),
6510 9 : loop_mask, vec_opmask[i], gsi);
6511 : else
6512 : mask = loop_mask;
6513 : }
6514 1253 : else if (is_cond_op)
6515 67 : mask = vec_opmask[i];
6516 1262 : if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
6517 : {
6518 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num, vectype_in,
6519 : i, 1, false);
6520 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
6521 0 : bias = build_int_cst (intQI_type_node, biasval);
6522 0 : if (!is_cond_op)
6523 0 : mask = build_minus_one_cst (truth_type_for (vectype_in));
6524 : }
6525 :
6526 : /* Handle MINUS by adding the negative. */
6527 1262 : if (reduc_fn != IFN_LAST && code == MINUS_EXPR)
6528 : {
6529 0 : tree negated = make_ssa_name (vectype_out);
6530 0 : new_stmt = gimple_build_assign (negated, NEGATE_EXPR, def0);
6531 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6532 0 : def0 = negated;
6533 : }
6534 :
6535 9 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
6536 1271 : && mask && mask_reduc_fn == IFN_LAST)
6537 9 : def0 = merge_with_identity (gsi, mask, vectype_out, def0,
6538 : vector_identity);
6539 :
6540 : /* On the first iteration the input is simply the scalar phi
6541 : result, and for subsequent iterations it is the output of
6542 : the preceding operation. */
6543 1262 : if (reduc_fn != IFN_LAST || (mask && mask_reduc_fn != IFN_LAST))
6544 : {
6545 0 : if (mask && len && mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6546 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 5, reduc_var,
6547 : def0, mask, len, bias);
6548 0 : else if (mask && mask_reduc_fn == IFN_MASK_FOLD_LEFT_PLUS)
6549 0 : new_stmt = gimple_build_call_internal (mask_reduc_fn, 3, reduc_var,
6550 : def0, mask);
6551 : else
6552 0 : new_stmt = gimple_build_call_internal (reduc_fn, 2, reduc_var,
6553 : def0);
6554 : /* For chained SLP reductions the output of the previous reduction
6555 : operation serves as the input of the next. For the final statement
6556 : the output cannot be a temporary - we reuse the original
6557 : scalar destination of the last statement. */
6558 0 : if (i != vec_num - 1)
6559 : {
6560 0 : gimple_set_lhs (new_stmt, scalar_dest_var);
6561 0 : reduc_var = make_ssa_name (scalar_dest_var, new_stmt);
6562 0 : gimple_set_lhs (new_stmt, reduc_var);
6563 : }
6564 : }
6565 : else
6566 : {
6567 1262 : reduc_var = vect_expand_fold_left (gsi, scalar_dest_var,
6568 : tree_code (code), reduc_var, def0,
6569 : mask);
6570 1262 : new_stmt = SSA_NAME_DEF_STMT (reduc_var);
6571 : /* Remove the statement, so that we can use the same code paths
6572 : as for statements that we've just created. */
6573 1262 : gimple_stmt_iterator tmp_gsi = gsi_for_stmt (new_stmt);
6574 1262 : gsi_remove (&tmp_gsi, true);
6575 : }
6576 :
6577 1262 : if (i == vec_num - 1)
6578 : {
6579 930 : gimple_set_lhs (new_stmt, scalar_dest);
6580 930 : vect_finish_replace_stmt (loop_vinfo,
6581 : scalar_dest_def_info,
6582 : new_stmt);
6583 : }
6584 : else
6585 332 : vect_finish_stmt_generation (loop_vinfo,
6586 : scalar_dest_def_info,
6587 : new_stmt, gsi);
6588 :
6589 1262 : slp_node->push_vec_def (new_stmt);
6590 : }
6591 :
6592 930 : return true;
6593 930 : }
6594 :
6595 : /* Function is_nonwrapping_integer_induction.
6596 :
6597 : Check if STMT_VINFO (which is part of loop LOOP) both increments and
6598 : does not cause overflow. */
6599 :
6600 : static bool
6601 414 : is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, class loop *loop)
6602 : {
6603 414 : gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
6604 414 : tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo);
6605 414 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo);
6606 414 : tree lhs_type = TREE_TYPE (gimple_phi_result (phi));
6607 414 : widest_int ni, max_loop_value, lhs_max;
6608 414 : wi::overflow_type overflow = wi::OVF_NONE;
6609 :
6610 : /* Make sure this is a regular induction. */
6611 414 : if (STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_vinfo) != vect_step_op_add)
6612 : return false;
6613 :
6614 : /* Make sure the loop is integer based. */
6615 408 : if (TREE_CODE (base) != INTEGER_CST
6616 109 : || TREE_CODE (step) != INTEGER_CST)
6617 : return false;
6618 :
6619 : /* Check that the max size of the loop will not wrap. */
6620 :
6621 109 : if (TYPE_OVERFLOW_UNDEFINED (lhs_type))
6622 : return true;
6623 :
6624 8 : if (! max_stmt_executions (loop, &ni))
6625 : return false;
6626 :
6627 8 : max_loop_value = wi::mul (wi::to_widest (step), ni, TYPE_SIGN (lhs_type),
6628 8 : &overflow);
6629 8 : if (overflow)
6630 : return false;
6631 :
6632 8 : max_loop_value = wi::add (wi::to_widest (base), max_loop_value,
6633 16 : TYPE_SIGN (lhs_type), &overflow);
6634 8 : if (overflow)
6635 : return false;
6636 :
6637 8 : return (wi::min_precision (max_loop_value, TYPE_SIGN (lhs_type))
6638 8 : <= TYPE_PRECISION (lhs_type));
6639 414 : }
6640 :
6641 : /* Check if masking can be supported by inserting a conditional expression.
6642 : CODE is the code for the operation. COND_FN is the conditional internal
6643 : function, if it exists. VECTYPE_IN is the type of the vector input. */
6644 : static bool
6645 6041 : use_mask_by_cond_expr_p (code_helper code, internal_fn cond_fn,
6646 : tree vectype_in)
6647 : {
6648 6041 : if (cond_fn != IFN_LAST
6649 6041 : && direct_internal_fn_supported_p (cond_fn, vectype_in,
6650 : OPTIMIZE_FOR_SPEED))
6651 : return false;
6652 :
6653 4318 : if (code.is_tree_code ())
6654 4298 : switch (tree_code (code))
6655 : {
6656 397 : case DOT_PROD_EXPR:
6657 397 : case SAD_EXPR:
6658 397 : return true;
6659 :
6660 : default:
6661 : break;
6662 : }
6663 : return false;
6664 : }
6665 :
6666 : /* Insert a conditional expression to enable masked vectorization. CODE is the
6667 : code for the operation. VOP is the array of operands. MASK is the loop
6668 : mask. GSI is a statement iterator used to place the new conditional
6669 : expression. */
6670 : static void
6671 4 : build_vect_cond_expr (code_helper code, tree vop[3], tree mask,
6672 : gimple_stmt_iterator *gsi)
6673 : {
6674 4 : switch (tree_code (code))
6675 : {
6676 4 : case DOT_PROD_EXPR:
6677 4 : {
6678 4 : tree vectype = TREE_TYPE (vop[1]);
6679 4 : tree zero = build_zero_cst (vectype);
6680 4 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6681 4 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6682 : mask, vop[1], zero);
6683 4 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6684 4 : vop[1] = masked_op1;
6685 4 : break;
6686 : }
6687 :
6688 0 : case SAD_EXPR:
6689 0 : {
6690 0 : tree vectype = TREE_TYPE (vop[1]);
6691 0 : tree masked_op1 = make_temp_ssa_name (vectype, NULL, "masked_op1");
6692 0 : gassign *select = gimple_build_assign (masked_op1, VEC_COND_EXPR,
6693 : mask, vop[1], vop[0]);
6694 0 : gsi_insert_before (gsi, select, GSI_SAME_STMT);
6695 0 : vop[1] = masked_op1;
6696 0 : break;
6697 : }
6698 :
6699 0 : default:
6700 0 : gcc_unreachable ();
6701 : }
6702 4 : }
6703 :
6704 : /* Given an operation with CODE in loop reduction path whose reduction PHI is
6705 : specified by REDUC_INFO, the operation has TYPE of scalar result, and its
6706 : input vectype is represented by VECTYPE_IN. The vectype of vectorized result
6707 : may be different from VECTYPE_IN, either in base type or vectype lanes,
6708 : lane-reducing operation is the case. This function check if it is possible,
6709 : and how to perform partial vectorization on the operation in the context
6710 : of LOOP_VINFO. It returns whether partial vectors are possible. The
6711 : caller is responsible to register the usage appropriately. */
6712 :
6713 : static bool
6714 4221 : vect_reduction_update_partial_vector_usage (loop_vec_info loop_vinfo,
6715 : vect_reduc_info reduc_info,
6716 : code_helper code, tree type,
6717 : tree vectype_in)
6718 : {
6719 4221 : enum vect_reduction_type reduc_type = VECT_REDUC_INFO_TYPE (reduc_info);
6720 4221 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
6721 4221 : internal_fn cond_fn
6722 1157 : = ((code.is_internal_fn ()
6723 1157 : && internal_fn_mask_index ((internal_fn)code) != -1)
6724 4221 : ? (internal_fn)code : get_conditional_internal_fn (code, type));
6725 :
6726 4221 : if (reduc_type != FOLD_LEFT_REDUCTION
6727 3401 : && !use_mask_by_cond_expr_p (code, cond_fn, vectype_in)
6728 7506 : && (cond_fn == IFN_LAST
6729 3285 : || !direct_internal_fn_supported_p (cond_fn, vectype_in,
6730 : OPTIMIZE_FOR_SPEED)))
6731 : {
6732 2071 : if (dump_enabled_p ())
6733 101 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6734 : "can't operate on partial vectors because"
6735 : " no conditional operation is available.\n");
6736 2071 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6737 : }
6738 2150 : else if (reduc_type == FOLD_LEFT_REDUCTION
6739 2150 : && reduc_fn == IFN_LAST
6740 2150 : && !expand_vec_cond_expr_p (vectype_in, truth_type_for (vectype_in)))
6741 : {
6742 0 : if (dump_enabled_p ())
6743 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6744 : "can't operate on partial vectors because"
6745 : " no conditional operation is available.\n");
6746 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6747 : }
6748 2150 : else if (reduc_type == FOLD_LEFT_REDUCTION
6749 820 : && internal_fn_mask_index (reduc_fn) == -1
6750 820 : && FLOAT_TYPE_P (vectype_in)
6751 2970 : && HONOR_SIGN_DEPENDENT_ROUNDING (vectype_in))
6752 : {
6753 0 : if (dump_enabled_p ())
6754 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6755 : "can't operate on partial vectors because"
6756 : " signed zeros cannot be preserved.\n");
6757 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6758 : }
6759 : else
6760 : return true;
6761 : return false;
6762 : }
6763 :
6764 : /* Check if STMT_INFO is a lane-reducing operation that can be vectorized in
6765 : the context of LOOP_VINFO, and vector cost will be recorded in COST_VEC,
6766 : and the analysis is for slp if SLP_NODE is not NULL.
6767 :
6768 : For a lane-reducing operation, the loop reduction path that it lies in,
6769 : may contain normal operation, or other lane-reducing operation of different
6770 : input type size, an example as:
6771 :
6772 : int sum = 0;
6773 : for (i)
6774 : {
6775 : ...
6776 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
6777 : sum += w[i]; // widen-sum <vector(16) char>
6778 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
6779 : sum += n[i]; // normal <vector(4) int>
6780 : ...
6781 : }
6782 :
6783 : Vectorization factor is essentially determined by operation whose input
6784 : vectype has the most lanes ("vector(16) char" in the example), while we
6785 : need to choose input vectype with the least lanes ("vector(4) int" in the
6786 : example) to determine effective number of vector reduction PHIs. */
6787 :
6788 : bool
6789 408192 : vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
6790 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
6791 : {
6792 408192 : gimple *stmt = stmt_info->stmt;
6793 :
6794 408192 : if (!lane_reducing_stmt_p (stmt))
6795 : return false;
6796 :
6797 740 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
6798 :
6799 740 : if (!INTEGRAL_TYPE_P (type))
6800 : return false;
6801 :
6802 : /* Do not try to vectorize bit-precision reductions. */
6803 740 : if (!type_has_mode_precision_p (type))
6804 : return false;
6805 :
6806 740 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
6807 :
6808 : /* TODO: Support lane-reducing operation that does not directly participate
6809 : in loop reduction. */
6810 740 : if (!reduc_info)
6811 : return false;
6812 :
6813 : /* Lane-reducing pattern inside any inner loop of LOOP_VINFO is not
6814 : recognized. */
6815 740 : gcc_assert (!nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info));
6816 740 : gcc_assert (VECT_REDUC_INFO_TYPE (reduc_info) == TREE_CODE_REDUCTION);
6817 :
6818 2960 : for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
6819 : {
6820 2220 : slp_tree slp_op;
6821 2220 : tree op;
6822 2220 : tree vectype;
6823 2220 : enum vect_def_type dt;
6824 :
6825 2220 : if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
6826 : &slp_op, &dt, &vectype))
6827 : {
6828 0 : if (dump_enabled_p ())
6829 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6830 : "use not simple.\n");
6831 0 : return false;
6832 : }
6833 :
6834 2220 : if (!vectype)
6835 : {
6836 15 : vectype = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op),
6837 : slp_op);
6838 15 : if (!vectype)
6839 : return false;
6840 : }
6841 :
6842 2220 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype))
6843 : {
6844 0 : if (dump_enabled_p ())
6845 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6846 : "incompatible vector types for invariants\n");
6847 : return false;
6848 : }
6849 :
6850 2220 : if (i == STMT_VINFO_REDUC_IDX (stmt_info))
6851 740 : continue;
6852 :
6853 : /* There should be at most one cycle def in the stmt. */
6854 1480 : if (VECTORIZABLE_CYCLE_DEF (dt))
6855 : return false;
6856 : }
6857 :
6858 740 : slp_tree node_in = SLP_TREE_CHILDREN (slp_node)[0];
6859 740 : tree vectype_in = SLP_TREE_VECTYPE (node_in);
6860 740 : gcc_assert (vectype_in);
6861 :
6862 : /* Compute number of effective vector statements for costing from the
6863 : number of input lanes allow for excess lanes in the last input vector. */
6864 740 : unsigned int ncopies_for_cost, excess_elts;
6865 740 : if (!vect_get_num_copies_for_invariant (loop_vinfo, node_in,
6866 : &ncopies_for_cost,
6867 : &excess_elts))
6868 : {
6869 0 : if (dump_enabled_p ())
6870 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6871 : "incompatible vector types for invariants\n");
6872 : return false;
6873 : }
6874 740 : gcc_assert (ncopies_for_cost >= 1);
6875 :
6876 740 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
6877 : {
6878 116 : enum tree_code code = gimple_assign_rhs_code (stmt);
6879 116 : if (vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
6880 116 : code, type, vectype_in))
6881 : {
6882 116 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
6883 116 : internal_fn mask_reduc_fn
6884 116 : = get_masked_reduction_fn (reduc_fn, vectype_in);
6885 116 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
6886 116 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
6887 116 : if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
6888 0 : vect_record_loop_len (loop_vinfo, lens, ncopies_for_cost,
6889 : vectype_in, 1);
6890 : else
6891 116 : vect_record_loop_mask (loop_vinfo, masks, ncopies_for_cost,
6892 : vectype_in, NULL);
6893 : }
6894 : }
6895 :
6896 740 : if (vect_is_emulated_mixed_dot_prod (slp_node))
6897 : {
6898 : /* We need extra two invariants: one that contains the minimum signed
6899 : value and one that contains half of its negative. */
6900 15 : int prologue_stmts = 2;
6901 15 : unsigned cost = record_stmt_cost (cost_vec, prologue_stmts,
6902 : scalar_to_vec, slp_node, 0,
6903 : vect_prologue);
6904 15 : if (dump_enabled_p ())
6905 0 : dump_printf (MSG_NOTE, "vectorizable_lane_reducing: "
6906 : "extra prologue_cost = %d .\n", cost);
6907 :
6908 : /* Three dot-products and a subtraction. */
6909 15 : ncopies_for_cost *= 4;
6910 : }
6911 :
6912 740 : record_stmt_cost (cost_vec, (int) ncopies_for_cost, vector_stmt, slp_node,
6913 : 0, vect_body);
6914 :
6915 : /* Transform via vect_transform_reduction. */
6916 740 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
6917 740 : return true;
6918 : }
6919 :
6920 : /* Function vectorizable_reduction.
6921 :
6922 : Check if STMT_INFO performs a reduction operation that can be vectorized.
6923 : If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6924 : stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6925 : Return true if STMT_INFO is vectorizable in this way.
6926 :
6927 : This function also handles reduction idioms (patterns) that have been
6928 : recognized in advance during vect_pattern_recog. In this case, STMT_INFO
6929 : may be of this form:
6930 : X = pattern_expr (arg0, arg1, ..., X)
6931 : and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
6932 : sequence that had been detected and replaced by the pattern-stmt
6933 : (STMT_INFO).
6934 :
6935 : This function also handles reduction of condition expressions, for example:
6936 : for (int i = 0; i < N; i++)
6937 : if (a[i] < value)
6938 : last = a[i];
6939 : This is handled by vectorising the loop and creating an additional vector
6940 : containing the loop indexes for which "a[i] < value" was true. In the
6941 : function epilogue this is reduced to a single max value and then used to
6942 : index into the vector of results.
6943 :
6944 : In some cases of reduction patterns, the type of the reduction variable X is
6945 : different than the type of the other arguments of STMT_INFO.
6946 : In such cases, the vectype that is used when transforming STMT_INFO into
6947 : a vector stmt is different than the vectype that is used to determine the
6948 : vectorization factor, because it consists of a different number of elements
6949 : than the actual number of elements that are being operated upon in parallel.
6950 :
6951 : For example, consider an accumulation of shorts into an int accumulator.
6952 : On some targets it's possible to vectorize this pattern operating on 8
6953 : shorts at a time (hence, the vectype for purposes of determining the
6954 : vectorization factor should be V8HI); on the other hand, the vectype that
6955 : is used to create the vector form is actually V4SI (the type of the result).
6956 :
6957 : Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
6958 : indicates what is the actual level of parallelism (V8HI in the example), so
6959 : that the right vectorization factor would be derived. This vectype
6960 : corresponds to the type of arguments to the reduction stmt, and should *NOT*
6961 : be used to create the vectorized stmt. The right vectype for the vectorized
6962 : stmt is obtained from the type of the result X:
6963 : get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
6964 :
6965 : This means that, contrary to "regular" reductions (or "regular" stmts in
6966 : general), the following equation:
6967 : STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
6968 : does *NOT* necessarily hold for reduction patterns. */
6969 :
6970 : bool
6971 407452 : vectorizable_reduction (loop_vec_info loop_vinfo,
6972 : stmt_vec_info stmt_info, slp_tree slp_node,
6973 : slp_instance slp_node_instance,
6974 : stmt_vector_for_cost *cost_vec)
6975 : {
6976 407452 : tree vectype_in = NULL_TREE;
6977 407452 : enum vect_def_type cond_reduc_dt = vect_unknown_def_type;
6978 407452 : stmt_vec_info cond_stmt_vinfo = NULL;
6979 407452 : int i;
6980 407452 : int ncopies;
6981 407452 : bool single_defuse_cycle = false;
6982 407452 : tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE;
6983 407452 : tree cond_reduc_val = NULL_TREE;
6984 :
6985 : /* Make sure it was already recognized as a reduction computation. */
6986 407452 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
6987 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def
6988 407452 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
6989 : return false;
6990 :
6991 : /* The reduction meta. */
6992 85370 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
6993 :
6994 85370 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
6995 : {
6996 1379 : gcc_assert (is_a <gphi *> (stmt_info->stmt));
6997 : /* We eventually need to set a vector type on invariant arguments. */
6998 : unsigned j;
6999 : slp_tree child;
7000 4129 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
7001 2758 : if (!vect_maybe_update_slp_op_vectype (child,
7002 : SLP_TREE_VECTYPE (slp_node)))
7003 : {
7004 0 : if (dump_enabled_p ())
7005 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7006 : "incompatible vector types for "
7007 : "invariants\n");
7008 : return false;
7009 : }
7010 2758 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
7011 2758 : && !useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
7012 : SLP_TREE_VECTYPE (child)))
7013 : {
7014 : /* With bools we can have mask and non-mask precision vectors
7015 : or different non-mask precisions. while pattern recog is
7016 : supposed to guarantee consistency here, we do not have
7017 : pattern stmts for PHIs (PR123316).
7018 : Deal with that here instead of ICEing later. */
7019 8 : if (dump_enabled_p ())
7020 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7021 : "incompatible vector type setup from "
7022 : "bool pattern detection\n");
7023 : return false;
7024 : }
7025 : /* Analysis for double-reduction is done on the outer
7026 : loop PHI, nested cycles have no further restrictions. */
7027 1371 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7028 1371 : return true;
7029 : }
7030 :
7031 83991 : if (!is_a <gphi *> (stmt_info->stmt))
7032 : {
7033 8055 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def);
7034 8055 : SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
7035 8055 : return true;
7036 : }
7037 :
7038 75936 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
7039 75936 : stmt_vec_info phi_info = stmt_info;
7040 75936 : bool double_reduc = false;
7041 75936 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
7042 : {
7043 : /* We arrive here for both the inner loop LC PHI and the
7044 : outer loop PHI. The latter is what we want to analyze the
7045 : reduction with. The LC PHI is handled by vectorizable_lc_phi. */
7046 322 : if (gimple_bb (stmt_info->stmt) != loop->header)
7047 0 : return false;
7048 :
7049 : /* Set loop and phi_info to the inner loop. */
7050 322 : use_operand_p use_p;
7051 322 : gimple *use_stmt;
7052 322 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
7053 : &use_p, &use_stmt);
7054 322 : gcc_assert (res);
7055 322 : phi_info = loop_vinfo->lookup_stmt (use_stmt);
7056 322 : loop = loop->inner;
7057 322 : double_reduc = true;
7058 : }
7059 :
7060 75936 : const bool reduc_chain = reduc_info->is_reduc_chain;
7061 75936 : slp_node_instance->reduc_phis = slp_node;
7062 : /* ??? We're leaving slp_node to point to the PHIs, we only
7063 : need it to get at the number of vector stmts which wasn't
7064 : yet initialized for the instance root. */
7065 :
7066 : /* PHIs should not participate in patterns. */
7067 75936 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7068 75936 : gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
7069 :
7070 : /* Verify following REDUC_IDX from the latch def leads us back to the PHI
7071 : and compute the reduction chain length. Discover the real
7072 : reduction operation stmt on the way (slp_for_stmt_info). */
7073 75936 : unsigned reduc_chain_length = 0;
7074 75936 : stmt_info = NULL;
7075 75936 : slp_tree slp_for_stmt_info = NULL;
7076 75936 : slp_tree vdef_slp = slp_node_instance->root;
7077 167736 : while (vdef_slp != slp_node)
7078 : {
7079 92892 : int reduc_idx = SLP_TREE_REDUC_IDX (vdef_slp);
7080 92892 : if (reduc_idx == -1)
7081 : {
7082 1084 : if (dump_enabled_p ())
7083 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7084 : "reduction chain broken by patterns.\n");
7085 1092 : return false;
7086 : }
7087 91808 : stmt_vec_info vdef = SLP_TREE_REPRESENTATIVE (vdef_slp);
7088 91808 : if (is_a <gphi *> (vdef->stmt))
7089 : {
7090 644 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7091 : /* Do not count PHIs towards the chain length. */
7092 644 : continue;
7093 : }
7094 91164 : gimple_match_op op;
7095 91164 : if (!gimple_extract_op (vdef->stmt, &op))
7096 : {
7097 0 : if (dump_enabled_p ())
7098 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7099 : "reduction chain includes unsupported"
7100 : " statement type.\n");
7101 : return false;
7102 : }
7103 91164 : if (CONVERT_EXPR_CODE_P (op.code))
7104 : {
7105 5404 : if (!tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0])))
7106 : {
7107 8 : if (dump_enabled_p ())
7108 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7109 : "conversion in the reduction chain.\n");
7110 : return false;
7111 : }
7112 5396 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
7113 : }
7114 : else
7115 : {
7116 : /* First non-conversion stmt. */
7117 85760 : if (!slp_for_stmt_info)
7118 74832 : slp_for_stmt_info = vdef_slp;
7119 :
7120 85760 : if (lane_reducing_op_p (op.code))
7121 : {
7122 : /* The last operand of lane-reducing operation is for
7123 : reduction. */
7124 740 : gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
7125 :
7126 740 : slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
7127 740 : tree vectype_op = SLP_TREE_VECTYPE (op_node);
7128 740 : tree type_op = TREE_TYPE (op.ops[0]);
7129 740 : if (!vectype_op)
7130 : {
7131 18 : vectype_op = get_vectype_for_scalar_type (loop_vinfo,
7132 : type_op);
7133 18 : if (!vectype_op
7134 18 : || !vect_maybe_update_slp_op_vectype (op_node,
7135 : vectype_op))
7136 : return false;
7137 : }
7138 :
7139 : /* To accommodate lane-reducing operations of mixed input
7140 : vectypes, choose input vectype with the least lanes for the
7141 : reduction PHI statement, which would result in the most
7142 : ncopies for vectorized reduction results. */
7143 740 : if (!vectype_in
7144 740 : || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in)))
7145 769 : < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
7146 : vectype_in = vectype_op;
7147 : }
7148 85020 : else if (!vectype_in)
7149 74121 : vectype_in = SLP_TREE_VECTYPE (slp_node);
7150 85760 : vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
7151 : }
7152 91156 : reduc_chain_length++;
7153 : }
7154 74844 : if (!slp_for_stmt_info)
7155 : {
7156 12 : if (dump_enabled_p ())
7157 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7158 : "only noop-conversions in the reduction chain.\n");
7159 : return false;
7160 : }
7161 74832 : stmt_info = SLP_TREE_REPRESENTATIVE (slp_for_stmt_info);
7162 :
7163 : /* PHIs should not participate in patterns. */
7164 74832 : gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
7165 :
7166 : /* 1. Is vectorizable reduction? */
7167 : /* Not supportable if the reduction variable is used in the loop, unless
7168 : it's a reduction chain. */
7169 74832 : if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer
7170 0 : && !reduc_chain)
7171 : return false;
7172 :
7173 : /* Reductions that are not used even in an enclosing outer-loop,
7174 : are expected to be "live" (used out of the loop). */
7175 74832 : if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
7176 0 : && !STMT_VINFO_LIVE_P (stmt_info))
7177 : return false;
7178 :
7179 : /* 2. Has this been recognized as a reduction pattern?
7180 :
7181 : Check if STMT represents a pattern that has been recognized
7182 : in earlier analysis stages. For stmts that represent a pattern,
7183 : the STMT_VINFO_RELATED_STMT field records the last stmt in
7184 : the original sequence that constitutes the pattern. */
7185 :
7186 74832 : stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
7187 74832 : if (orig_stmt_info)
7188 : {
7189 5121 : gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
7190 5121 : gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
7191 : }
7192 :
7193 : /* 3. Check the operands of the operation. The first operands are defined
7194 : inside the loop body. The last operand is the reduction variable,
7195 : which is defined by the loop-header-phi. */
7196 :
7197 74832 : tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
7198 74832 : VECT_REDUC_INFO_VECTYPE (reduc_info) = vectype_out;
7199 :
7200 74832 : gimple_match_op op;
7201 74832 : if (!gimple_extract_op (stmt_info->stmt, &op))
7202 0 : gcc_unreachable ();
7203 74832 : bool lane_reducing = lane_reducing_op_p (op.code);
7204 :
7205 74832 : if (!POINTER_TYPE_P (op.type) && !INTEGRAL_TYPE_P (op.type)
7206 22288 : && !SCALAR_FLOAT_TYPE_P (op.type))
7207 : return false;
7208 :
7209 : /* Do not try to vectorize bit-precision reductions. */
7210 74832 : if (!type_has_mode_precision_p (op.type)
7211 1789 : && op.code != BIT_AND_EXPR
7212 1659 : && op.code != BIT_IOR_EXPR
7213 75324 : && op.code != BIT_XOR_EXPR)
7214 : return false;
7215 :
7216 : /* Lane-reducing ops also never can be used in a SLP reduction group
7217 : since we'll mix lanes belonging to different reductions. But it's
7218 : OK to use them in a reduction chain or when the reduction group
7219 : has just one element. */
7220 74522 : if (lane_reducing
7221 74522 : && !reduc_chain
7222 668 : && SLP_TREE_LANES (slp_node) > 1)
7223 : {
7224 0 : if (dump_enabled_p ())
7225 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7226 : "lane-reducing reduction in reduction group.\n");
7227 : return false;
7228 : }
7229 :
7230 : /* We'll verify the reduction operation only later - avoid
7231 : all operations that mismatch on the number of SLP children. */
7232 149044 : if (op.num_ops != SLP_TREE_CHILDREN (slp_for_stmt_info).length ())
7233 : {
7234 0 : if (dump_enabled_p ())
7235 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7236 : "unsupported reduction operation.\n");
7237 : return false;
7238 : }
7239 :
7240 : /* All uses but the last are expected to be defined in the loop.
7241 : The last use is the reduction variable. In case of nested cycle this
7242 : assumption is not true: we use reduc_index to record the index of the
7243 : reduction variable. */
7244 74522 : slp_tree *slp_op = XALLOCAVEC (slp_tree, op.num_ops);
7245 74522 : tree *vectype_op = XALLOCAVEC (tree, op.num_ops);
7246 74522 : gcc_assert (op.code != COND_EXPR || !COMPARISON_CLASS_P (op.ops[0]));
7247 238671 : for (i = 0; i < (int) op.num_ops; i++)
7248 : {
7249 : /* The condition of COND_EXPR is checked in vectorizable_condition(). */
7250 164149 : if (i == 0 && op.code == COND_EXPR)
7251 82160 : continue;
7252 :
7253 163270 : stmt_vec_info def_stmt_info;
7254 163270 : enum vect_def_type dt;
7255 163270 : if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
7256 : i, &op.ops[i], &slp_op[i], &dt,
7257 163270 : &vectype_op[i], &def_stmt_info))
7258 : {
7259 0 : if (dump_enabled_p ())
7260 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7261 : "use not simple.\n");
7262 0 : return false;
7263 : }
7264 :
7265 : /* Skip reduction operands, and for an IFN_COND_OP we might hit the
7266 : reduction operand twice (once as definition, once as else). */
7267 163270 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]
7268 326540 : == SLP_TREE_CHILDREN
7269 163270 : (slp_for_stmt_info)[SLP_TREE_REDUC_IDX (slp_for_stmt_info)])
7270 81281 : continue;
7271 :
7272 : /* There should be only one cycle def in the stmt, the one
7273 : leading to reduc_def. */
7274 81989 : if (SLP_TREE_CHILDREN (slp_for_stmt_info)[i]->cycle_info.id != -1)
7275 : return false;
7276 :
7277 81989 : if (!vectype_op[i])
7278 7443 : vectype_op[i]
7279 7443 : = get_vectype_for_scalar_type (loop_vinfo,
7280 7443 : TREE_TYPE (op.ops[i]), slp_op[i]);
7281 :
7282 : /* Record how the non-reduction-def value of COND_EXPR is defined.
7283 : ??? For a chain of multiple CONDs we'd have to match them up all. */
7284 81989 : if (op.code == COND_EXPR && reduc_chain_length == 1)
7285 : {
7286 856 : if (dt == vect_constant_def)
7287 : {
7288 124 : cond_reduc_dt = dt;
7289 124 : cond_reduc_val = op.ops[i];
7290 : }
7291 732 : else if (dt == vect_induction_def
7292 414 : && def_stmt_info
7293 1146 : && is_nonwrapping_integer_induction (def_stmt_info, loop))
7294 : {
7295 109 : cond_reduc_dt = dt;
7296 109 : cond_stmt_vinfo = def_stmt_info;
7297 : }
7298 : }
7299 : }
7300 :
7301 74522 : enum vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7302 : /* If we have a condition reduction, see if we can simplify it further. */
7303 74522 : if (reduction_type == COND_REDUCTION)
7304 : {
7305 867 : if (SLP_TREE_LANES (slp_node) != 1)
7306 : return false;
7307 :
7308 : /* When the condition uses the reduction value in the condition, fail. */
7309 843 : if (SLP_TREE_REDUC_IDX (slp_node) == 0)
7310 : {
7311 0 : if (dump_enabled_p ())
7312 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7313 : "condition depends on previous iteration\n");
7314 : return false;
7315 : }
7316 :
7317 843 : if (reduc_chain_length == 1
7318 843 : && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST, vectype_in,
7319 : OPTIMIZE_FOR_SPEED)
7320 820 : || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
7321 : vectype_in,
7322 : OPTIMIZE_FOR_SPEED)))
7323 : {
7324 0 : if (dump_enabled_p ())
7325 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7326 : "optimizing condition reduction with"
7327 : " FOLD_EXTRACT_LAST.\n");
7328 0 : VECT_REDUC_INFO_TYPE (reduc_info) = EXTRACT_LAST_REDUCTION;
7329 : }
7330 843 : else if (cond_reduc_dt == vect_induction_def)
7331 : {
7332 109 : tree base
7333 : = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo);
7334 109 : tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo);
7335 :
7336 109 : gcc_assert (TREE_CODE (base) == INTEGER_CST
7337 : && TREE_CODE (step) == INTEGER_CST);
7338 109 : cond_reduc_val = NULL_TREE;
7339 109 : enum tree_code cond_reduc_op_code = ERROR_MARK;
7340 109 : tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
7341 109 : if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
7342 : ;
7343 : /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
7344 : above base; punt if base is the minimum value of the type for
7345 : MAX_EXPR or maximum value of the type for MIN_EXPR for now. */
7346 97 : else if (tree_int_cst_sgn (step) == -1)
7347 : {
7348 18 : cond_reduc_op_code = MIN_EXPR;
7349 18 : if (tree_int_cst_sgn (base) == -1)
7350 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7351 18 : else if (tree_int_cst_lt (base,
7352 18 : TYPE_MAX_VALUE (TREE_TYPE (base))))
7353 18 : cond_reduc_val
7354 18 : = int_const_binop (PLUS_EXPR, base, integer_one_node);
7355 : }
7356 : else
7357 : {
7358 79 : cond_reduc_op_code = MAX_EXPR;
7359 79 : if (tree_int_cst_sgn (base) == 1)
7360 0 : cond_reduc_val = build_int_cst (TREE_TYPE (base), 0);
7361 79 : else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base)),
7362 : base))
7363 79 : cond_reduc_val
7364 79 : = int_const_binop (MINUS_EXPR, base, integer_one_node);
7365 : }
7366 97 : if (cond_reduc_val)
7367 : {
7368 97 : if (dump_enabled_p ())
7369 61 : dump_printf_loc (MSG_NOTE, vect_location,
7370 : "condition expression based on "
7371 : "integer induction.\n");
7372 97 : VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_op_code;
7373 97 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info)
7374 97 : = cond_reduc_val;
7375 97 : VECT_REDUC_INFO_TYPE (reduc_info) = INTEGER_INDUC_COND_REDUCTION;
7376 : }
7377 : }
7378 734 : else if (cond_reduc_dt == vect_constant_def)
7379 : {
7380 114 : enum vect_def_type cond_initial_dt;
7381 114 : tree cond_initial_val = vect_phi_initial_value (reduc_def_phi);
7382 114 : vect_is_simple_use (cond_initial_val, loop_vinfo, &cond_initial_dt);
7383 114 : if (cond_initial_dt == vect_constant_def
7384 145 : && types_compatible_p (TREE_TYPE (cond_initial_val),
7385 31 : TREE_TYPE (cond_reduc_val)))
7386 : {
7387 31 : tree lt, gt;
7388 31 : enum tree_code cond_reduc_code = ERROR_MARK;
7389 31 : if ((lt = fold_binary (LT_EXPR, boolean_type_node,
7390 : cond_initial_val, cond_reduc_val))
7391 31 : && integer_onep (lt))
7392 : cond_reduc_code = MAX_EXPR;
7393 13 : else if ((gt = fold_binary (GT_EXPR, boolean_type_node,
7394 : cond_initial_val, cond_reduc_val))
7395 13 : && integer_onep (gt))
7396 : cond_reduc_code = MIN_EXPR;
7397 : if (cond_reduc_code != ERROR_MARK)
7398 : {
7399 25 : if (dump_enabled_p ())
7400 16 : dump_printf_loc (MSG_NOTE, vect_location,
7401 : "condition expression based on "
7402 : "compile time constant.\n");
7403 : /* Record reduction code at analysis stage. */
7404 25 : VECT_REDUC_INFO_CODE (reduc_info) = cond_reduc_code;
7405 25 : VECT_REDUC_INFO_TYPE (reduc_info) = CONST_COND_REDUCTION;
7406 : }
7407 : }
7408 : }
7409 : }
7410 :
7411 74498 : if (STMT_VINFO_LIVE_P (phi_info))
7412 : return false;
7413 :
7414 74498 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
7415 :
7416 74498 : gcc_assert (ncopies >= 1);
7417 :
7418 74498 : poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
7419 :
7420 : /* 4.2. Check support for the epilog operation.
7421 :
7422 : If STMT represents a reduction pattern, then the type of the
7423 : reduction variable may be different than the type of the rest
7424 : of the arguments. For example, consider the case of accumulation
7425 : of shorts into an int accumulator; The original code:
7426 : S1: int_a = (int) short_a;
7427 : orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
7428 :
7429 : was replaced with:
7430 : STMT: int_acc = widen_sum <short_a, int_acc>
7431 :
7432 : This means that:
7433 : 1. The tree-code that is used to create the vector operation in the
7434 : epilog code (that reduces the partial results) is not the
7435 : tree-code of STMT, but is rather the tree-code of the original
7436 : stmt from the pattern that STMT is replacing. I.e, in the example
7437 : above we want to use 'widen_sum' in the loop, but 'plus' in the
7438 : epilog.
7439 : 2. The type (mode) we use to check available target support
7440 : for the vector operation to be created in the *epilog*, is
7441 : determined by the type of the reduction variable (in the example
7442 : above we'd check this: optab_handler (plus_optab, vect_int_mode])).
7443 : However the type (mode) we use to check available target support
7444 : for the vector operation to be created *inside the loop*, is
7445 : determined by the type of the other arguments to STMT (in the
7446 : example we'd check this: optab_handler (widen_sum_optab,
7447 : vect_short_mode)).
7448 :
7449 : This is contrary to "regular" reductions, in which the types of all
7450 : the arguments are the same as the type of the reduction variable.
7451 : For "regular" reductions we can therefore use the same vector type
7452 : (and also the same tree-code) when generating the epilog code and
7453 : when generating the code inside the loop. */
7454 :
7455 74498 : code_helper orig_code = VECT_REDUC_INFO_CODE (reduc_info);
7456 :
7457 : /* If conversion might have created a conditional operation like
7458 : IFN_COND_ADD already. Use the internal code for the following checks. */
7459 74498 : if (orig_code.is_internal_fn ())
7460 : {
7461 6835 : tree_code new_code = conditional_internal_fn_code (internal_fn (orig_code));
7462 6835 : orig_code = new_code != ERROR_MARK ? new_code : orig_code;
7463 : }
7464 :
7465 74498 : VECT_REDUC_INFO_CODE (reduc_info) = orig_code;
7466 :
7467 74498 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
7468 74498 : if (reduction_type == TREE_CODE_REDUCTION)
7469 : {
7470 : /* Check whether it's ok to change the order of the computation.
7471 : Generally, when vectorizing a reduction we change the order of the
7472 : computation. This may change the behavior of the program in some
7473 : cases, so we need to check that this is ok. One exception is when
7474 : vectorizing an outer-loop: the inner-loop is executed sequentially,
7475 : and therefore vectorizing reductions in the inner-loop during
7476 : outer-loop vectorization is safe. Likewise when we are vectorizing
7477 : a series of reductions using SLP and the VF is one the reductions
7478 : are performed in scalar order. */
7479 73655 : if (!reduc_chain
7480 73655 : && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1u))
7481 : ;
7482 73483 : else if (needs_fold_left_reduction_p (op.type, orig_code))
7483 : {
7484 : /* When vectorizing a reduction chain w/o SLP the reduction PHI
7485 : is not directly used in stmt. */
7486 5307 : if (reduc_chain_length != 1)
7487 : {
7488 97 : if (dump_enabled_p ())
7489 20 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7490 : "in-order reduction chain without SLP.\n");
7491 : return false;
7492 : }
7493 : /* Code generation doesn't support function calls other
7494 : than .COND_*. */
7495 5210 : if (!op.code.is_tree_code ()
7496 5398 : && !(op.code.is_internal_fn ()
7497 94 : && conditional_internal_fn_code (internal_fn (op.code))
7498 : != ERROR_MARK))
7499 : {
7500 18 : if (dump_enabled_p ())
7501 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7502 : "in-order reduction chain operation not "
7503 : "supported.\n");
7504 : return false;
7505 : }
7506 5192 : VECT_REDUC_INFO_TYPE (reduc_info)
7507 5192 : = reduction_type = FOLD_LEFT_REDUCTION;
7508 : }
7509 68176 : else if (!commutative_binary_op_p (orig_code, op.type)
7510 68176 : || !associative_binary_op_p (orig_code, op.type))
7511 : {
7512 172 : if (dump_enabled_p ())
7513 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7514 : "reduction: not commutative/associative\n");
7515 : return false;
7516 : }
7517 : }
7518 :
7519 5192 : if ((reduction_type == COND_REDUCTION
7520 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7521 : || reduction_type == CONST_COND_REDUCTION
7522 69019 : || reduction_type == EXTRACT_LAST_REDUCTION)
7523 843 : && ncopies > 1)
7524 : {
7525 276 : if (dump_enabled_p ())
7526 60 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7527 : "multiple types in condition reduction.\n");
7528 : return false;
7529 : }
7530 :
7531 : /* See if we can convert a mask vector to a corresponding bool data vector
7532 : to perform the epilogue reduction. */
7533 73935 : tree alt_vectype_out = NULL_TREE;
7534 73935 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out))
7535 : {
7536 1185 : alt_vectype_out
7537 2370 : = get_related_vectype_for_scalar_type (loop_vinfo->vector_mode,
7538 1185 : TREE_TYPE (vectype_out),
7539 : TYPE_VECTOR_SUBPARTS
7540 : (vectype_out));
7541 1185 : if (!alt_vectype_out
7542 1185 : || maybe_ne (TYPE_VECTOR_SUBPARTS (alt_vectype_out),
7543 2370 : TYPE_VECTOR_SUBPARTS (vectype_out))
7544 2370 : || !expand_vec_cond_expr_p (alt_vectype_out, vectype_out))
7545 : alt_vectype_out = NULL_TREE;
7546 : }
7547 :
7548 73935 : internal_fn reduc_fn = IFN_LAST;
7549 73935 : if (reduction_type == TREE_CODE_REDUCTION
7550 73935 : || reduction_type == FOLD_LEFT_REDUCTION
7551 : || reduction_type == INTEGER_INDUC_COND_REDUCTION
7552 567 : || reduction_type == CONST_COND_REDUCTION)
7553 : {
7554 68290 : if (reduction_type == FOLD_LEFT_REDUCTION
7555 77888 : ? fold_left_reduction_fn (orig_code, &reduc_fn)
7556 68290 : : reduction_fn_for_scalar_code (orig_code, &reduc_fn))
7557 : {
7558 72810 : internal_fn sbool_fn = IFN_LAST;
7559 72810 : if (reduc_fn == IFN_LAST)
7560 : ;
7561 70714 : else if ((!VECTOR_BOOLEAN_TYPE_P (vectype_out)
7562 1185 : || (GET_MODE_CLASS (TYPE_MODE (vectype_out))
7563 : == MODE_VECTOR_BOOL))
7564 140243 : && direct_internal_fn_supported_p (reduc_fn, vectype_out,
7565 : OPTIMIZE_FOR_SPEED))
7566 : ;
7567 18836 : else if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
7568 1185 : && sbool_reduction_fn_for_fn (reduc_fn, &sbool_fn)
7569 20021 : && direct_internal_fn_supported_p (sbool_fn, vectype_out,
7570 : OPTIMIZE_FOR_SPEED))
7571 131 : reduc_fn = sbool_fn;
7572 18705 : else if (reduction_type != FOLD_LEFT_REDUCTION
7573 18705 : && alt_vectype_out
7574 18705 : && direct_internal_fn_supported_p (reduc_fn, alt_vectype_out,
7575 : OPTIMIZE_FOR_SPEED))
7576 809 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7577 : else
7578 : {
7579 17896 : if (dump_enabled_p ())
7580 954 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7581 : "reduc op not supported by target.\n");
7582 :
7583 17896 : reduc_fn = IFN_LAST;
7584 : }
7585 : }
7586 : else
7587 : {
7588 672 : if (dump_enabled_p ())
7589 48 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7590 : "no reduc code for scalar code.\n");
7591 :
7592 : return false;
7593 : }
7594 72810 : if (reduc_fn == IFN_LAST
7595 72810 : && VECTOR_BOOLEAN_TYPE_P (vectype_out))
7596 : {
7597 245 : if (!alt_vectype_out)
7598 : {
7599 12 : if (dump_enabled_p ())
7600 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7601 : "cannot turn mask into bool data vector for "
7602 : "reduction epilogue.\n");
7603 : return false;
7604 : }
7605 233 : VECT_REDUC_INFO_VECTYPE_FOR_MASK (reduc_info) = alt_vectype_out;
7606 : }
7607 : }
7608 453 : else if (reduction_type == COND_REDUCTION)
7609 : {
7610 453 : int scalar_precision
7611 453 : = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op.type));
7612 453 : cr_index_scalar_type = make_unsigned_type (scalar_precision);
7613 453 : cr_index_vector_type = get_same_sized_vectype (cr_index_scalar_type,
7614 : vectype_out);
7615 :
7616 453 : if (direct_internal_fn_supported_p (IFN_REDUC_MAX, cr_index_vector_type,
7617 : OPTIMIZE_FOR_SPEED))
7618 22 : reduc_fn = IFN_REDUC_MAX;
7619 : }
7620 73251 : VECT_REDUC_INFO_FN (reduc_info) = reduc_fn;
7621 :
7622 73251 : if (reduction_type != EXTRACT_LAST_REDUCTION
7623 : && reduc_fn == IFN_LAST
7624 : && !nunits_out.is_constant ())
7625 : {
7626 : if (dump_enabled_p ())
7627 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7628 : "missing target support for reduction on"
7629 : " variable-length vectors.\n");
7630 : return false;
7631 : }
7632 :
7633 : /* For SLP reductions, see if there is a neutral value we can use. */
7634 73251 : tree neutral_op = NULL_TREE;
7635 73251 : tree initial_value = NULL_TREE;
7636 73251 : if (reduc_chain)
7637 2291 : initial_value = vect_phi_initial_value (reduc_def_phi);
7638 73251 : neutral_op = neutral_op_for_reduction (TREE_TYPE
7639 : (gimple_phi_result (reduc_def_phi)),
7640 : orig_code, initial_value);
7641 73251 : VECT_REDUC_INFO_NEUTRAL_OP (reduc_info) = neutral_op;
7642 :
7643 73251 : if (double_reduc && reduction_type == FOLD_LEFT_REDUCTION)
7644 : {
7645 : /* We can't support in-order reductions of code such as this:
7646 :
7647 : for (int i = 0; i < n1; ++i)
7648 : for (int j = 0; j < n2; ++j)
7649 : l += a[j];
7650 :
7651 : since GCC effectively transforms the loop when vectorizing:
7652 :
7653 : for (int i = 0; i < n1 / VF; ++i)
7654 : for (int j = 0; j < n2; ++j)
7655 : for (int k = 0; k < VF; ++k)
7656 : l += a[j];
7657 :
7658 : which is a reassociation of the original operation. */
7659 66 : if (dump_enabled_p ())
7660 20 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7661 : "in-order double reduction not supported.\n");
7662 :
7663 : return false;
7664 : }
7665 :
7666 73185 : if (reduction_type == FOLD_LEFT_REDUCTION
7667 4454 : && SLP_TREE_LANES (slp_node) > 1
7668 194 : && !reduc_chain)
7669 : {
7670 : /* We cannot use in-order reductions in this case because there is
7671 : an implicit reassociation of the operations involved. */
7672 72 : if (dump_enabled_p ())
7673 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7674 : "in-order unchained SLP reductions not supported.\n");
7675 : return false;
7676 : }
7677 :
7678 : /* For double reductions, and for SLP reductions with a neutral value,
7679 : we construct a variable-length initial vector by loading a vector
7680 : full of the neutral value and then shift-and-inserting the start
7681 : values into the low-numbered elements. This is however not needed
7682 : when neutral and initial value are equal or we can handle the
7683 : initial value via adjustment in the epilogue. */
7684 73113 : if ((double_reduc || neutral_op)
7685 : && !nunits_out.is_constant ()
7686 : && reduction_type != INTEGER_INDUC_COND_REDUCTION
7687 : && !((SLP_TREE_LANES (slp_node) == 1 || reduc_chain)
7688 : && neutral_op
7689 : && (!double_reduc
7690 : || operand_equal_p (neutral_op,
7691 : vect_phi_initial_value (reduc_def_phi))))
7692 : && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT,
7693 : vectype_out, OPTIMIZE_FOR_BOTH))
7694 : {
7695 : if (dump_enabled_p ())
7696 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7697 : "reduction on variable-length vectors requires"
7698 : " target support for a vector-shift-and-insert"
7699 : " operation.\n");
7700 : return false;
7701 : }
7702 :
7703 : /* Check extra constraints for variable-length unchained SLP reductions. */
7704 73113 : if (!reduc_chain
7705 : && !nunits_out.is_constant ())
7706 : {
7707 : /* We checked above that we could build the initial vector when
7708 : there's a neutral element value. Check here for the case in
7709 : which each SLP statement has its own initial value and in which
7710 : that value needs to be repeated for every instance of the
7711 : statement within the initial vector. */
7712 : unsigned int group_size = SLP_TREE_LANES (slp_node);
7713 : if (!neutral_op
7714 : && !can_duplicate_and_interleave_p (loop_vinfo, group_size,
7715 : TREE_TYPE (vectype_out)))
7716 : {
7717 : if (dump_enabled_p ())
7718 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7719 : "unsupported form of SLP reduction for"
7720 : " variable-length vectors: cannot build"
7721 : " initial vector.\n");
7722 : return false;
7723 : }
7724 : /* The epilogue code relies on the number of elements being a multiple
7725 : of the group size. The duplicate-and-interleave approach to setting
7726 : up the initial vector does too. */
7727 : if (!multiple_p (nunits_out, group_size))
7728 : {
7729 : if (dump_enabled_p ())
7730 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7731 : "unsupported form of SLP reduction for"
7732 : " variable-length vectors: the vector size"
7733 : " is not a multiple of the number of results.\n");
7734 : return false;
7735 : }
7736 : }
7737 :
7738 73113 : if (reduction_type == COND_REDUCTION)
7739 : {
7740 453 : widest_int ni;
7741 :
7742 453 : if (! max_loop_iterations (loop, &ni))
7743 : {
7744 14 : if (dump_enabled_p ())
7745 0 : dump_printf_loc (MSG_NOTE, vect_location,
7746 : "loop count not known, cannot create cond "
7747 : "reduction.\n");
7748 : return false;
7749 : }
7750 : /* Convert backedges to iterations. */
7751 439 : ni += 1;
7752 :
7753 : /* The additional index will be the same type as the condition. Check
7754 : that the loop can fit into this less one (because we'll use up the
7755 : zero slot for when there are no matches). */
7756 439 : tree max_index = TYPE_MAX_VALUE (cr_index_scalar_type);
7757 439 : if (wi::geu_p (ni, wi::to_widest (max_index)))
7758 : {
7759 90 : if (dump_enabled_p ())
7760 54 : dump_printf_loc (MSG_NOTE, vect_location,
7761 : "loop size is greater than data size.\n");
7762 : return false;
7763 : }
7764 453 : }
7765 :
7766 : /* In case the vectorization factor (VF) is bigger than the number
7767 : of elements that we can fit in a vectype (nunits), we have to generate
7768 : more than one vector stmt - i.e - we need to "unroll" the
7769 : vector stmt by a factor VF/nunits. For more details see documentation
7770 : in vectorizable_operation. */
7771 :
7772 : /* If the reduction is used in an outer loop we need to generate
7773 : VF intermediate results, like so (e.g. for ncopies=2):
7774 : r0 = phi (init, r0)
7775 : r1 = phi (init, r1)
7776 : r0 = x0 + r0;
7777 : r1 = x1 + r1;
7778 : (i.e. we generate VF results in 2 registers).
7779 : In this case we have a separate def-use cycle for each copy, and therefore
7780 : for each copy we get the vector def for the reduction variable from the
7781 : respective phi node created for this copy.
7782 :
7783 : Otherwise (the reduction is unused in the loop nest), we can combine
7784 : together intermediate results, like so (e.g. for ncopies=2):
7785 : r = phi (init, r)
7786 : r = x0 + r;
7787 : r = x1 + r;
7788 : (i.e. we generate VF/2 results in a single register).
7789 : In this case for each copy we get the vector def for the reduction variable
7790 : from the vectorized reduction operation generated in the previous iteration.
7791 :
7792 : This only works when we see both the reduction PHI and its only consumer
7793 : in vectorizable_reduction and there are no intermediate stmts
7794 : participating. When unrolling we want each unrolled iteration to have its
7795 : own reduction accumulator since one of the main goals of unrolling a
7796 : reduction is to reduce the aggregate loop-carried latency. */
7797 73009 : if (ncopies > 1
7798 73009 : && !reduc_chain
7799 8152 : && SLP_TREE_LANES (slp_node) == 1
7800 7984 : && (STMT_VINFO_RELEVANT (stmt_info) <= vect_used_only_live)
7801 7961 : && reduc_chain_length == 1
7802 7557 : && loop_vinfo->suggested_unroll_factor == 1)
7803 73009 : single_defuse_cycle = true;
7804 :
7805 73009 : if (single_defuse_cycle && !lane_reducing)
7806 : {
7807 6600 : gcc_assert (op.code != COND_EXPR);
7808 :
7809 : /* 4. check support for the operation in the loop
7810 :
7811 : This isn't necessary for the lane reduction codes, since they
7812 : can only be produced by pattern matching, and it's up to the
7813 : pattern matcher to test for support. The main reason for
7814 : specifically skipping this step is to avoid rechecking whether
7815 : mixed-sign dot-products can be implemented using signed
7816 : dot-products. */
7817 6600 : machine_mode vec_mode = TYPE_MODE (vectype_in);
7818 6600 : if (!directly_supported_p (op.code, vectype_in, optab_vector))
7819 : {
7820 2076 : if (dump_enabled_p ())
7821 36 : dump_printf (MSG_NOTE, "op not supported by target.\n");
7822 4152 : if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
7823 2076 : || !vect_can_vectorize_without_simd_p (op.code))
7824 : single_defuse_cycle = false;
7825 : else
7826 10 : if (dump_enabled_p ())
7827 0 : dump_printf (MSG_NOTE, "proceeding using word mode.\n");
7828 : }
7829 :
7830 6600 : if (vect_emulated_vector_p (vectype_in)
7831 6600 : && !vect_can_vectorize_without_simd_p (op.code))
7832 : {
7833 0 : if (dump_enabled_p ())
7834 0 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
7835 : return false;
7836 : }
7837 : }
7838 73009 : if (dump_enabled_p () && single_defuse_cycle)
7839 719 : dump_printf_loc (MSG_NOTE, vect_location,
7840 : "using single def-use cycle for reduction by reducing "
7841 : "multiple vectors to one in the loop body\n");
7842 73009 : VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info) = single_defuse_cycle;
7843 :
7844 : /* For lane-reducing operation, the below processing related to single
7845 : defuse-cycle will be done in its own vectorizable function. One more
7846 : thing to note is that the operation must not be involved in fold-left
7847 : reduction. */
7848 73009 : single_defuse_cycle &= !lane_reducing;
7849 :
7850 73009 : if (single_defuse_cycle || reduction_type == FOLD_LEFT_REDUCTION)
7851 28853 : for (i = 0; i < (int) op.num_ops; i++)
7852 20028 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], vectype_op[i]))
7853 : {
7854 0 : if (dump_enabled_p ())
7855 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7856 : "incompatible vector types for invariants\n");
7857 : return false;
7858 : }
7859 :
7860 73009 : vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
7861 : reduction_type, ncopies, cost_vec);
7862 : /* Cost the reduction op inside the loop if transformed via
7863 : vect_transform_reduction for non-lane-reducing operation. Otherwise
7864 : this is costed by the separate vectorizable_* routines. */
7865 73009 : if (single_defuse_cycle)
7866 4534 : record_stmt_cost (cost_vec, ncopies, vector_stmt,
7867 : slp_for_stmt_info, 0, vect_body);
7868 :
7869 73009 : if (dump_enabled_p ()
7870 73009 : && reduction_type == FOLD_LEFT_REDUCTION)
7871 262 : dump_printf_loc (MSG_NOTE, vect_location,
7872 : "using an in-order (fold-left) reduction.\n");
7873 73009 : SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
7874 :
7875 : /* All but single defuse-cycle optimized and fold-left reductions go
7876 : through their own vectorizable_* routines. */
7877 73009 : stmt_vec_info tem
7878 73009 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (slp_node_instance));
7879 73009 : if (!single_defuse_cycle && reduction_type != FOLD_LEFT_REDUCTION)
7880 64184 : STMT_VINFO_DEF_TYPE (tem) = vect_internal_def;
7881 : else
7882 : {
7883 8825 : STMT_VINFO_DEF_TYPE (tem) = vect_reduction_def;
7884 8825 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
7885 8825 : && vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info,
7886 : op.code, op.type,
7887 : vectype_in))
7888 : {
7889 2034 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
7890 2034 : internal_fn mask_reduc_fn
7891 2034 : = get_masked_reduction_fn (reduc_fn, vectype_in);
7892 2034 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
7893 2034 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
7894 2034 : if (mask_reduc_fn == IFN_MASK_LEN_FOLD_LEFT_PLUS)
7895 0 : vect_record_loop_len (loop_vinfo, lens, ncopies, vectype_in, 1);
7896 : else
7897 2034 : vect_record_loop_mask (loop_vinfo, masks, ncopies,
7898 : vectype_in, NULL);
7899 : }
7900 : }
7901 : return true;
7902 : }
7903 :
7904 : /* STMT_INFO is a dot-product reduction whose multiplication operands
7905 : have different signs. Emit a sequence to emulate the operation
7906 : using a series of signed DOT_PROD_EXPRs and return the last
7907 : statement generated. VEC_DEST is the result of the vector operation
7908 : and VOP lists its inputs. */
7909 :
7910 : static gassign *
7911 4 : vect_emulate_mixed_dot_prod (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
7912 : gimple_stmt_iterator *gsi, tree vec_dest,
7913 : tree vop[3])
7914 : {
7915 4 : tree wide_vectype = signed_type_for (TREE_TYPE (vec_dest));
7916 4 : tree narrow_vectype = signed_type_for (TREE_TYPE (vop[0]));
7917 4 : tree narrow_elttype = TREE_TYPE (narrow_vectype);
7918 4 : gimple *new_stmt;
7919 :
7920 : /* Make VOP[0] the unsigned operand VOP[1] the signed operand. */
7921 4 : if (!TYPE_UNSIGNED (TREE_TYPE (vop[0])))
7922 0 : std::swap (vop[0], vop[1]);
7923 :
7924 : /* Convert all inputs to signed types. */
7925 12 : for (int i = 1; i < 3; ++i)
7926 8 : if (TYPE_UNSIGNED (TREE_TYPE (vop[i])))
7927 : {
7928 0 : tree tmp = make_ssa_name (signed_type_for (TREE_TYPE (vop[i])));
7929 0 : new_stmt = gimple_build_assign (tmp, NOP_EXPR, vop[i]);
7930 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7931 0 : vop[i] = tmp;
7932 : }
7933 :
7934 : /* In the comments below we assume 8-bit inputs for simplicity,
7935 : but the approach works for any full integer type. */
7936 :
7937 : /* Create a vector of -128. */
7938 4 : tree min_narrow_elttype = TYPE_MIN_VALUE (narrow_elttype);
7939 4 : tree min_narrow = build_vector_from_val (TREE_TYPE (vop[0]),
7940 4 : fold_convert
7941 : (TREE_TYPE (TREE_TYPE (vop[0])),
7942 : min_narrow_elttype));
7943 :
7944 : /* Create a vector of 64. */
7945 4 : auto half_wi = wi::lrshift (wi::to_wide (min_narrow_elttype), 1);
7946 4 : tree half_narrow = wide_int_to_tree (narrow_elttype, half_wi);
7947 4 : half_narrow = build_vector_from_val (narrow_vectype, half_narrow);
7948 :
7949 : /* Emit: SUB_RES = VOP[0] - 128 in an unsigned type. */
7950 4 : tree sub_res = make_ssa_name (TREE_TYPE (vop[0]));
7951 4 : new_stmt = gimple_build_assign (sub_res, PLUS_EXPR, vop[0], min_narrow);
7952 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7953 :
7954 4 : vop[0] = make_ssa_name (narrow_vectype);
7955 4 : new_stmt = gimple_build_assign (vop[0], VIEW_CONVERT_EXPR,
7956 : build1 (VIEW_CONVERT_EXPR, narrow_vectype,
7957 : sub_res));
7958 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7959 :
7960 : /* Emit:
7961 :
7962 : STAGE1 = DOT_PROD_EXPR <VOP[1], 64, VOP[2]>;
7963 : STAGE2 = DOT_PROD_EXPR <VOP[1], 64, STAGE1>;
7964 : STAGE3 = DOT_PROD_EXPR <SUB_RES, -128, STAGE2>;
7965 :
7966 : on the basis that x * y == (x - 128) * y + 64 * y + 64 * y
7967 : Doing the two 64 * y steps first allows more time to compute x. */
7968 4 : tree stage1 = make_ssa_name (wide_vectype);
7969 4 : new_stmt = gimple_build_assign (stage1, DOT_PROD_EXPR,
7970 : vop[1], half_narrow, vop[2]);
7971 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7972 :
7973 4 : tree stage2 = make_ssa_name (wide_vectype);
7974 4 : new_stmt = gimple_build_assign (stage2, DOT_PROD_EXPR,
7975 : vop[1], half_narrow, stage1);
7976 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7977 :
7978 4 : tree stage3 = make_ssa_name (wide_vectype);
7979 4 : new_stmt = gimple_build_assign (stage3, DOT_PROD_EXPR,
7980 : vop[0], vop[1], stage2);
7981 4 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
7982 :
7983 : /* Convert STAGE3 to the reduction type. */
7984 4 : return gimple_build_assign (vec_dest, CONVERT_EXPR, stage3);
7985 4 : }
7986 :
7987 : /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
7988 : value. */
7989 :
7990 : bool
7991 2640 : vect_transform_reduction (loop_vec_info loop_vinfo,
7992 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
7993 : slp_tree slp_node)
7994 : {
7995 2640 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
7996 2640 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
7997 :
7998 2640 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
7999 :
8000 2640 : if (nested_in_vect_loop_p (loop, stmt_info))
8001 : {
8002 0 : loop = loop->inner;
8003 0 : gcc_assert (VECT_REDUC_INFO_DEF_TYPE (reduc_info)
8004 : == vect_double_reduction_def);
8005 : }
8006 :
8007 2640 : gimple_match_op op;
8008 2640 : if (!gimple_extract_op (stmt_info->stmt, &op))
8009 0 : gcc_unreachable ();
8010 :
8011 : /* All uses but the last are expected to be defined in the loop.
8012 : The last use is the reduction variable. In case of nested cycle this
8013 : assumption is not true: we use reduc_index to record the index of the
8014 : reduction variable. */
8015 2640 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
8016 2640 : tree vectype_in = SLP_TREE_VECTYPE (slp_node);
8017 2640 : if (lane_reducing_op_p (op.code))
8018 281 : vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]);
8019 :
8020 2640 : code_helper code = canonicalize_code (op.code, op.type);
8021 2640 : internal_fn cond_fn
8022 482 : = ((code.is_internal_fn ()
8023 482 : && internal_fn_mask_index ((internal_fn)code) != -1)
8024 2640 : ? (internal_fn)code : get_conditional_internal_fn (code, op.type));
8025 :
8026 2640 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
8027 2640 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
8028 2640 : bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
8029 :
8030 : /* Transform. */
8031 2640 : tree new_temp = NULL_TREE;
8032 18480 : auto_vec<tree> vec_oprnds[3];
8033 :
8034 2640 : if (dump_enabled_p ())
8035 795 : dump_printf_loc (MSG_NOTE, vect_location, "transform reduction.\n");
8036 :
8037 : /* A binary COND_OP reduction must have the same definition and else
8038 : value. */
8039 3122 : bool cond_fn_p = code.is_internal_fn ()
8040 482 : && conditional_internal_fn_code (internal_fn (code)) != ERROR_MARK;
8041 482 : if (cond_fn_p)
8042 : {
8043 482 : gcc_assert (code == IFN_COND_ADD || code == IFN_COND_SUB
8044 : || code == IFN_COND_MUL || code == IFN_COND_AND
8045 : || code == IFN_COND_IOR || code == IFN_COND_XOR
8046 : || code == IFN_COND_MIN || code == IFN_COND_MAX);
8047 482 : gcc_assert (op.num_ops == 4
8048 : && (op.ops[reduc_index]
8049 : == op.ops[internal_fn_else_index ((internal_fn) code)]));
8050 : }
8051 :
8052 2640 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
8053 :
8054 2640 : vect_reduction_type reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
8055 2640 : if (reduction_type == FOLD_LEFT_REDUCTION)
8056 : {
8057 930 : internal_fn reduc_fn = VECT_REDUC_INFO_FN (reduc_info);
8058 930 : gcc_assert (code.is_tree_code () || cond_fn_p);
8059 930 : return vectorize_fold_left_reduction
8060 930 : (loop_vinfo, stmt_info, gsi, slp_node,
8061 930 : code, reduc_fn, op.num_ops, vectype_in,
8062 930 : reduc_index, masks, lens);
8063 : }
8064 :
8065 1710 : bool single_defuse_cycle = VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info);
8066 1710 : bool lane_reducing = lane_reducing_op_p (code);
8067 1429 : gcc_assert (single_defuse_cycle || lane_reducing);
8068 :
8069 1710 : if (lane_reducing)
8070 : {
8071 : /* The last operand of lane-reducing op is for reduction. */
8072 281 : gcc_assert (reduc_index == (int) op.num_ops - 1);
8073 : }
8074 :
8075 : /* Create the destination vector */
8076 1710 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
8077 1710 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
8078 :
8079 : /* Get NCOPIES vector definitions for all operands except the reduction
8080 : definition. */
8081 1710 : if (!cond_fn_p)
8082 : {
8083 1257 : gcc_assert (reduc_index >= 0 && reduc_index <= 2);
8084 1257 : vect_get_vec_defs (loop_vinfo, slp_node,
8085 1257 : single_defuse_cycle && reduc_index == 0
8086 1257 : ? NULL_TREE : op.ops[0], &vec_oprnds[0],
8087 1257 : single_defuse_cycle && reduc_index == 1
8088 1257 : ? NULL_TREE : op.ops[1], &vec_oprnds[1],
8089 1257 : op.num_ops == 3
8090 281 : && !(single_defuse_cycle && reduc_index == 2)
8091 1364 : ? op.ops[2] : NULL_TREE, &vec_oprnds[2]);
8092 : }
8093 : else
8094 : {
8095 : /* For a conditional operation pass the truth type as mask
8096 : vectype. */
8097 453 : gcc_assert (single_defuse_cycle
8098 : && (reduc_index == 1 || reduc_index == 2));
8099 453 : vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
8100 : &vec_oprnds[0],
8101 2 : reduc_index == 1 ? NULL_TREE : op.ops[1],
8102 : &vec_oprnds[1],
8103 453 : reduc_index == 2 ? NULL_TREE : op.ops[2],
8104 : &vec_oprnds[2]);
8105 : }
8106 :
8107 : /* For single def-use cycles get one copy of the vectorized reduction
8108 : definition. */
8109 3000 : unsigned vec_in_num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
8110 1710 : if (single_defuse_cycle)
8111 : {
8112 1603 : vect_get_vec_defs (loop_vinfo, slp_node,
8113 1603 : reduc_index == 0 ? op.ops[0] : NULL_TREE,
8114 : &vec_oprnds[0],
8115 1603 : reduc_index == 1 ? op.ops[1] : NULL_TREE,
8116 : &vec_oprnds[1],
8117 1603 : reduc_index == 2 ? op.ops[2] : NULL_TREE,
8118 : &vec_oprnds[2]);
8119 : }
8120 107 : else if (lane_reducing)
8121 : {
8122 : /* For normal reduction, consistency between vectorized def/use is
8123 : naturally ensured when mapping from scalar statement. But if lane-
8124 : reducing op is involved in reduction, thing would become somewhat
8125 : complicated in that the op's result and operand for accumulation are
8126 : limited to less lanes than other operands, which certainly causes
8127 : def/use mismatch on adjacent statements around the op if do not have
8128 : any kind of specific adjustment. One approach is to refit lane-
8129 : reducing op in the way of introducing new trivial pass-through copies
8130 : to fix possible def/use gap, so as to make it behave like a normal op.
8131 : And vector reduction PHIs are always generated to the full extent, no
8132 : matter lane-reducing op exists or not. If some copies or PHIs are
8133 : actually superfluous, they would be cleaned up by passes after
8134 : vectorization. An example for single-lane slp, lane-reducing ops
8135 : with mixed input vectypes in a reduction chain, is given as below.
8136 : Similarly, this handling is applicable for multiple-lane slp as well.
8137 :
8138 : int sum = 1;
8139 : for (i)
8140 : {
8141 : sum += d0[i] * d1[i]; // dot-prod <vector(16) char>
8142 : sum += w[i]; // widen-sum <vector(16) char>
8143 : sum += abs(s0[i] - s1[i]); // sad <vector(8) short>
8144 : sum += n[i]; // normal <vector(4) int>
8145 : }
8146 :
8147 : The vector size is 128-bit，vectorization factor is 16. Reduction
8148 : statements would be transformed as:
8149 :
8150 : vector<4> int sum_v0 = { 0, 0, 0, 1 };
8151 : vector<4> int sum_v1 = { 0, 0, 0, 0 };
8152 : vector<4> int sum_v2 = { 0, 0, 0, 0 };
8153 : vector<4> int sum_v3 = { 0, 0, 0, 0 };
8154 :
8155 : for (i / 16)
8156 : {
8157 : sum_v0 = DOT_PROD (d0_v0[i: 0 ~ 15], d1_v0[i: 0 ~ 15], sum_v0);
8158 : sum_v1 = sum_v1; // copy
8159 : sum_v2 = sum_v2; // copy
8160 : sum_v3 = sum_v3; // copy
8161 :
8162 : sum_v0 = sum_v0; // copy
8163 : sum_v1 = WIDEN_SUM (w_v1[i: 0 ~ 15], sum_v1);
8164 : sum_v2 = sum_v2; // copy
8165 : sum_v3 = sum_v3; // copy
8166 :
8167 : sum_v0 = sum_v0; // copy
8168 : sum_v1 = SAD (s0_v1[i: 0 ~ 7 ], s1_v1[i: 0 ~ 7 ], sum_v1);
8169 : sum_v2 = SAD (s0_v2[i: 8 ~ 15], s1_v2[i: 8 ~ 15], sum_v2);
8170 : sum_v3 = sum_v3; // copy
8171 :
8172 : sum_v0 += n_v0[i: 0 ~ 3 ];
8173 : sum_v1 += n_v1[i: 4 ~ 7 ];
8174 : sum_v2 += n_v2[i: 8 ~ 11];
8175 : sum_v3 += n_v3[i: 12 ~ 15];
8176 : }
8177 :
8178 : Moreover, for a higher instruction parallelism in final vectorized
8179 : loop, it is considered to make those effective vector lane-reducing
8180 : ops be distributed evenly among all def-use cycles. In the above
8181 : example, DOT_PROD, WIDEN_SUM and SADs are generated into disparate
8182 : cycles, instruction dependency among them could be eliminated. */
8183 107 : unsigned effec_ncopies = vec_oprnds[0].length ();
8184 107 : unsigned total_ncopies = vec_oprnds[reduc_index].length ();
8185 :
8186 107 : gcc_assert (effec_ncopies <= total_ncopies);
8187 :
8188 107 : if (effec_ncopies < total_ncopies)
8189 : {
8190 294 : for (unsigned i = 0; i < op.num_ops - 1; i++)
8191 : {
8192 392 : gcc_assert (vec_oprnds[i].length () == effec_ncopies);
8193 196 : vec_oprnds[i].safe_grow_cleared (total_ncopies);
8194 : }
8195 : }
8196 :
8197 107 : tree reduc_vectype_in = vectype_in;
8198 107 : gcc_assert (reduc_vectype_in);
8199 : }
8200 :
8201 1710 : bool emulated_mixed_dot_prod = vect_is_emulated_mixed_dot_prod (slp_node);
8202 1710 : unsigned num = vec_oprnds[reduc_index == 0 ? 1 : 0].length ();
8203 1710 : unsigned mask_index = 0;
8204 :
8205 7469 : for (unsigned i = 0; i < num; ++i)
8206 : {
8207 5759 : gimple *new_stmt;
8208 5759 : tree vop[3] = { vec_oprnds[0][i], vec_oprnds[1][i], NULL_TREE };
8209 5759 : if (!vop[0] || !vop[1])
8210 : {
8211 503 : tree reduc_vop = vec_oprnds[reduc_index][i];
8212 :
8213 : /* If could not generate an effective vector statement for current
8214 : portion of reduction operand, insert a trivial copy to simply
8215 : handle over the operand to other dependent statements. */
8216 503 : gcc_assert (reduc_vop);
8217 :
8218 503 : if (TREE_CODE (reduc_vop) == SSA_NAME
8219 503 : && !SSA_NAME_IS_DEFAULT_DEF (reduc_vop))
8220 503 : new_stmt = SSA_NAME_DEF_STMT (reduc_vop);
8221 : else
8222 : {
8223 0 : new_temp = make_ssa_name (vec_dest);
8224 0 : new_stmt = gimple_build_assign (new_temp, reduc_vop);
8225 0 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
8226 : gsi);
8227 : }
8228 : }
8229 5256 : else if (masked_loop_p && !mask_by_cond_expr)
8230 : {
8231 : /* No conditional ifns have been defined for lane-reducing op
8232 : yet. */
8233 16 : gcc_assert (!lane_reducing);
8234 :
8235 16 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8236 : vec_in_num, vectype_in,
8237 : mask_index++);
8238 16 : gcall *call;
8239 24 : if (code.is_internal_fn () && cond_fn_p)
8240 : {
8241 16 : gcc_assert (op.num_ops >= 3
8242 : && internal_fn_mask_index (internal_fn (code)) == 0);
8243 8 : vop[2] = vec_oprnds[2][i];
8244 8 : mask = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask),
8245 : mask, vop[0], gsi);
8246 8 : call = gimple_build_call_internal (cond_fn, 4, mask, vop[1],
8247 : vop[2], vop[reduc_index]);
8248 : }
8249 : else
8250 8 : call = gimple_build_call_internal (cond_fn, 4, mask, vop[0],
8251 : vop[1], vop[reduc_index]);
8252 16 : new_temp = make_ssa_name (vec_dest, call);
8253 16 : gimple_call_set_lhs (call, new_temp);
8254 16 : gimple_call_set_nothrow (call, true);
8255 16 : vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi);
8256 16 : new_stmt = call;
8257 : }
8258 : else
8259 : {
8260 5240 : if (op.num_ops >= 3)
8261 1792 : vop[2] = vec_oprnds[2][i];
8262 :
8263 5240 : if (masked_loop_p && mask_by_cond_expr)
8264 : {
8265 4 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
8266 : vec_in_num, vectype_in,
8267 : mask_index++);
8268 4 : build_vect_cond_expr (code, vop, mask, gsi);
8269 : }
8270 :
8271 5240 : if (emulated_mixed_dot_prod)
8272 4 : new_stmt = vect_emulate_mixed_dot_prod (loop_vinfo, stmt_info, gsi,
8273 : vec_dest, vop);
8274 :
8275 6578 : else if (code.is_internal_fn () && !cond_fn_p)
8276 0 : new_stmt = gimple_build_call_internal (internal_fn (code),
8277 : op.num_ops,
8278 : vop[0], vop[1], vop[2]);
8279 6578 : else if (code.is_internal_fn () && cond_fn_p)
8280 1342 : new_stmt = gimple_build_call_internal (internal_fn (code),
8281 : op.num_ops,
8282 : vop[0], vop[1], vop[2],
8283 : vop[reduc_index]);
8284 : else
8285 3894 : new_stmt = gimple_build_assign (vec_dest, tree_code (op.code),
8286 : vop[0], vop[1], vop[2]);
8287 5240 : new_temp = make_ssa_name (vec_dest, new_stmt);
8288 5240 : gimple_set_lhs (new_stmt, new_temp);
8289 5240 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi);
8290 : }
8291 :
8292 5759 : if (single_defuse_cycle && i < num - 1)
8293 3387 : vec_oprnds[reduc_index].safe_push (gimple_get_lhs (new_stmt));
8294 : else
8295 2372 : slp_node->push_vec_def (new_stmt);
8296 : }
8297 :
8298 : return true;
8299 10560 : }
8300 :
8301 : /* Transform phase of a cycle PHI. */
8302 :
8303 : bool
8304 23620 : vect_transform_cycle_phi (loop_vec_info loop_vinfo,
8305 : stmt_vec_info stmt_info,
8306 : slp_tree slp_node, slp_instance slp_node_instance)
8307 : {
8308 23620 : tree vectype_out = SLP_TREE_VECTYPE (slp_node);
8309 23620 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
8310 23620 : int i;
8311 23620 : bool nested_cycle = false;
8312 23620 : int vec_num;
8313 :
8314 23620 : if (nested_in_vect_loop_p (loop, stmt_info))
8315 : {
8316 23620 : loop = loop->inner;
8317 23620 : nested_cycle = true;
8318 : }
8319 :
8320 23620 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
8321 23620 : if (reduc_info
8322 23018 : && (VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
8323 23018 : || VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION))
8324 : /* Leave the scalar phi in place. */
8325 : return true;
8326 :
8327 22088 : if (reduc_info && reduc_info->is_reduc_chain && dump_enabled_p ())
8328 133 : dump_printf_loc (MSG_NOTE, vect_location,
8329 : "vectorizing a reduction chain\n");
8330 :
8331 22690 : vec_num = vect_get_num_copies (loop_vinfo, slp_node);
8332 :
8333 : /* Check whether we should use a single PHI node and accumulate
8334 : vectors to one before the backedge. */
8335 22690 : if (reduc_info && VECT_REDUC_INFO_FORCE_SINGLE_CYCLE (reduc_info))
8336 22690 : vec_num = 1;
8337 :
8338 : /* Create the destination vector */
8339 22690 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
8340 22690 : tree vec_dest = vect_create_destination_var (gimple_phi_result (phi),
8341 : vectype_out);
8342 :
8343 : /* Get the loop-entry arguments. */
8344 22690 : auto_vec<tree> vec_initial_defs;
8345 22690 : vec_initial_defs.reserve (vec_num);
8346 : /* Optimize: if initial_def is for REDUC_MAX smaller than the base
8347 : and we can't use zero for induc_val, use initial_def. Similarly
8348 : for REDUC_MIN and initial_def larger than the base. */
8349 22690 : if (reduc_info
8350 22088 : && VECT_REDUC_INFO_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
8351 : {
8352 62 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8353 62 : tree initial_def = vect_phi_initial_value (phi);
8354 62 : VECT_REDUC_INFO_INITIAL_VALUES (reduc_info).safe_push (initial_def);
8355 62 : tree induc_val = VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info);
8356 62 : if (TREE_CODE (initial_def) == INTEGER_CST
8357 60 : && !integer_zerop (induc_val)
8358 122 : && ((VECT_REDUC_INFO_CODE (reduc_info) == MAX_EXPR
8359 42 : && tree_int_cst_lt (initial_def, induc_val))
8360 58 : || (VECT_REDUC_INFO_CODE (reduc_info) == MIN_EXPR
8361 18 : && tree_int_cst_lt (induc_val, initial_def))))
8362 : {
8363 2 : induc_val = initial_def;
8364 : /* Communicate we used the initial_def to epilouge
8365 : generation. */
8366 2 : VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
8367 : }
8368 62 : vec_initial_defs.quick_push
8369 62 : (build_vector_from_val (vectype_out, induc_val));
8370 62 : }
8371 22628 : else if (nested_cycle)
8372 : {
8373 688 : unsigned phi_idx = loop_preheader_edge (loop)->dest_idx;
8374 688 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[phi_idx],
8375 : &vec_initial_defs);
8376 : }
8377 : else
8378 : {
8379 21940 : gcc_assert (slp_node == slp_node_instance->reduc_phis);
8380 21940 : vec<tree> &initial_values = VECT_REDUC_INFO_INITIAL_VALUES (reduc_info);
8381 21940 : vec<stmt_vec_info> &stmts = SLP_TREE_SCALAR_STMTS (slp_node);
8382 :
8383 21940 : unsigned int num_phis = stmts.length ();
8384 21940 : if (reduc_info->is_reduc_chain)
8385 216 : num_phis = 1;
8386 21940 : initial_values.reserve (num_phis);
8387 66290 : for (unsigned int i = 0; i < num_phis; ++i)
8388 : {
8389 22410 : gphi *this_phi = as_a<gphi *> (stmts[i]->stmt);
8390 22410 : initial_values.quick_push (vect_phi_initial_value (this_phi));
8391 : }
8392 21940 : tree neutral_op = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
8393 21940 : if (vec_num == 1
8394 21940 : && vect_find_reusable_accumulator (loop_vinfo,
8395 : reduc_info, vectype_out))
8396 : ;
8397 : /* Try to simplify the vector initialization by applying an
8398 : adjustment after the reduction has been performed. This
8399 : can also break a critical path but on the other hand
8400 : requires to keep the initial value live across the loop. */
8401 18005 : else if (neutral_op
8402 17413 : && initial_values.length () == 1
8403 17204 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
8404 35132 : && !operand_equal_p (neutral_op, initial_values[0]))
8405 : {
8406 12137 : VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT (reduc_info)
8407 12137 : = initial_values[0];
8408 12137 : initial_values[0] = neutral_op;
8409 : }
8410 21940 : if (!VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info)
8411 3935 : || loop_vinfo->main_loop_edge)
8412 43582 : get_initial_defs_for_reduction (loop_vinfo, reduc_info, vectype_out,
8413 : &vec_initial_defs, vec_num,
8414 : stmts.length (), neutral_op);
8415 : }
8416 :
8417 22690 : if (reduc_info)
8418 22088 : if (auto *accumulator = VECT_REDUC_INFO_REUSED_ACCUMULATOR (reduc_info))
8419 : {
8420 3935 : tree def = accumulator->reduc_input;
8421 3935 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8422 : {
8423 3932 : unsigned int nreduc;
8424 7864 : bool res = constant_multiple_p (TYPE_VECTOR_SUBPARTS
8425 3932 : (TREE_TYPE (def)),
8426 3932 : TYPE_VECTOR_SUBPARTS (vectype_out),
8427 : &nreduc);
8428 0 : gcc_assert (res);
8429 3932 : gimple_seq stmts = NULL;
8430 : /* Reduce the single vector to a smaller one. */
8431 3932 : if (nreduc != 1)
8432 : {
8433 : /* Perform the reduction in the appropriate type. */
8434 3932 : tree rvectype = vectype_out;
8435 3932 : if (!useless_type_conversion_p (TREE_TYPE (vectype_out),
8436 3932 : TREE_TYPE (TREE_TYPE (def))))
8437 235 : rvectype = build_vector_type (TREE_TYPE (TREE_TYPE (def)),
8438 : TYPE_VECTOR_SUBPARTS
8439 470 : (vectype_out));
8440 3932 : def = vect_create_partial_epilog (def, rvectype,
8441 : VECT_REDUC_INFO_CODE
8442 : (reduc_info),
8443 : &stmts);
8444 : }
8445 : /* The epilogue loop might use a different vector mode, like
8446 : VNx2DI vs. V2DI. */
8447 3932 : if (TYPE_MODE (vectype_out) != TYPE_MODE (TREE_TYPE (def)))
8448 : {
8449 0 : tree reduc_type = build_vector_type_for_mode
8450 0 : (TREE_TYPE (TREE_TYPE (def)), TYPE_MODE (vectype_out));
8451 0 : def = gimple_convert (&stmts, reduc_type, def);
8452 : }
8453 : /* Adjust the input so we pick up the partially reduced value
8454 : for the skip edge in vect_create_epilog_for_reduction. */
8455 3932 : accumulator->reduc_input = def;
8456 : /* And the reduction could be carried out using a different sign. */
8457 3932 : if (!useless_type_conversion_p (vectype_out, TREE_TYPE (def)))
8458 235 : def = gimple_convert (&stmts, vectype_out, def);
8459 3932 : edge e;
8460 3932 : if ((e = loop_vinfo->main_loop_edge)
8461 3932 : || (e = loop_vinfo->skip_this_loop_edge))
8462 : {
8463 : /* While we'd like to insert on the edge this will split
8464 : blocks and disturb bookkeeping, we also will eventually
8465 : need this on the skip edge. Rely on sinking to
8466 : fixup optimal placement and insert in the pred. */
8467 3783 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
8468 : /* Insert before a cond that eventually skips the
8469 : epilogue. */
8470 3783 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
8471 3766 : gsi_prev (&gsi);
8472 3783 : gsi_insert_seq_after (&gsi, stmts, GSI_CONTINUE_LINKING);
8473 : }
8474 : else
8475 149 : gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop),
8476 : stmts);
8477 : }
8478 3935 : if (loop_vinfo->main_loop_edge)
8479 3786 : vec_initial_defs[0]
8480 3786 : = vect_get_main_loop_result (loop_vinfo, def,
8481 3786 : vec_initial_defs[0]);
8482 : else
8483 149 : vec_initial_defs.safe_push (def);
8484 : }
8485 :
8486 : /* Generate the reduction PHIs upfront. */
8487 47242 : for (i = 0; i < vec_num; i++)
8488 : {
8489 24552 : tree vec_init_def = vec_initial_defs[i];
8490 : /* Create the reduction-phi that defines the reduction
8491 : operand. */
8492 24552 : gphi *new_phi = create_phi_node (vec_dest, loop->header);
8493 24552 : add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop),
8494 : UNKNOWN_LOCATION);
8495 :
8496 : /* The loop-latch arg is set in epilogue processing. */
8497 :
8498 24552 : slp_node->push_vec_def (new_phi);
8499 : }
8500 :
8501 22690 : return true;
8502 22690 : }
8503 :
8504 : /* Vectorizes LC PHIs. */
8505 :
8506 : bool
8507 198989 : vectorizable_lc_phi (loop_vec_info loop_vinfo,
8508 : stmt_vec_info stmt_info,
8509 : slp_tree slp_node)
8510 : {
8511 198989 : if (!loop_vinfo
8512 198989 : || !is_a <gphi *> (stmt_info->stmt)
8513 238623 : || gimple_phi_num_args (stmt_info->stmt) != 1)
8514 : return false;
8515 :
8516 792 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8517 0 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
8518 : return false;
8519 :
8520 : /* Deal with copies from externs or constants that disguise as
8521 : loop-closed PHI nodes (PR97886). */
8522 792 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
8523 : SLP_TREE_VECTYPE (slp_node)))
8524 : {
8525 0 : if (dump_enabled_p ())
8526 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8527 : "incompatible vector types for invariants\n");
8528 : return false;
8529 : }
8530 :
8531 : /* ??? This can happen with data vs. mask uses of boolean. */
8532 792 : if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
8533 792 : SLP_TREE_VECTYPE
8534 : (SLP_TREE_CHILDREN (slp_node)[0])))
8535 : {
8536 0 : if (dump_enabled_p ())
8537 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8538 : "missed mask promotion\n");
8539 : return false;
8540 : }
8541 :
8542 792 : SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
8543 792 : return true;
8544 : }
8545 :
8546 : bool
8547 516 : vect_transform_lc_phi (loop_vec_info loop_vinfo,
8548 : stmt_vec_info stmt_info,
8549 : slp_tree slp_node)
8550 : {
8551 :
8552 516 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8553 516 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8554 516 : basic_block bb = gimple_bb (stmt_info->stmt);
8555 516 : edge e = single_pred_edge (bb);
8556 516 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8557 516 : auto_vec<tree> vec_oprnds;
8558 516 : vect_get_vec_defs (loop_vinfo, slp_node, true, &vec_oprnds);
8559 1663 : for (unsigned i = 0; i < vec_oprnds.length (); i++)
8560 : {
8561 : /* Create the vectorized LC PHI node. */
8562 631 : gphi *new_phi = create_phi_node (vec_dest, bb);
8563 631 : add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION);
8564 631 : slp_node->push_vec_def (new_phi);
8565 : }
8566 :
8567 516 : return true;
8568 516 : }
8569 :
8570 : /* Vectorizes PHIs. */
8571 :
8572 : bool
8573 155121 : vectorizable_phi (bb_vec_info vinfo,
8574 : stmt_vec_info stmt_info,
8575 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8576 : {
8577 155121 : if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
8578 : return false;
8579 :
8580 73685 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
8581 : return false;
8582 :
8583 73685 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8584 :
8585 73685 : if (cost_vec) /* transformation not required. */
8586 : {
8587 : slp_tree child;
8588 : unsigned i;
8589 205702 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), i, child)
8590 144818 : if (!child)
8591 : {
8592 0 : if (dump_enabled_p ())
8593 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8594 : "PHI node with unvectorized backedge def\n");
8595 : return false;
8596 : }
8597 144818 : else if (!vect_maybe_update_slp_op_vectype (child, vectype))
8598 : {
8599 26 : if (dump_enabled_p ())
8600 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8601 : "incompatible vector types for invariants\n");
8602 : return false;
8603 : }
8604 144792 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8605 144792 : && !useless_type_conversion_p (vectype,
8606 : SLP_TREE_VECTYPE (child)))
8607 : {
8608 : /* With bools we can have mask and non-mask precision vectors
8609 : or different non-mask precisions. while pattern recog is
8610 : supposed to guarantee consistency here bugs in it can cause
8611 : mismatches (PR103489 and PR103800 for example).
8612 : Deal with them here instead of ICEing later. */
8613 18 : if (dump_enabled_p ())
8614 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8615 : "incompatible vector type setup from "
8616 : "bool pattern detection\n");
8617 : return false;
8618 : }
8619 :
8620 : /* For single-argument PHIs assume coalescing which means zero cost
8621 : for the scalar and the vector PHIs. This avoids artificially
8622 : favoring the vector path (but may pessimize it in some cases). */
8623 60884 : if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
8624 55105 : record_stmt_cost (cost_vec, vect_get_num_copies (vinfo, slp_node),
8625 : vector_stmt, slp_node, vectype, 0, vect_body);
8626 60884 : SLP_TREE_TYPE (slp_node) = phi_info_type;
8627 60884 : return true;
8628 : }
8629 :
8630 12757 : tree scalar_dest = gimple_phi_result (stmt_info->stmt);
8631 12757 : basic_block bb = gimple_bb (stmt_info->stmt);
8632 12757 : tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
8633 12757 : auto_vec<gphi *> new_phis;
8634 45014 : for (unsigned i = 0; i < gimple_phi_num_args (stmt_info->stmt); ++i)
8635 : {
8636 32257 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
8637 :
8638 : /* Skip not yet vectorized defs. */
8639 32866 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
8640 32257 : && SLP_TREE_VEC_DEFS (child).is_empty ())
8641 609 : continue;
8642 :
8643 31648 : auto_vec<tree> vec_oprnds;
8644 31648 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[i], &vec_oprnds);
8645 31648 : if (!new_phis.exists ())
8646 : {
8647 12757 : new_phis.create (vec_oprnds.length ());
8648 39631 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8649 : {
8650 : /* Create the vectorized LC PHI node. */
8651 14117 : new_phis.quick_push (create_phi_node (vec_dest, bb));
8652 14117 : slp_node->push_vec_def (new_phis[j]);
8653 : }
8654 : }
8655 31648 : edge e = gimple_phi_arg_edge (as_a <gphi *> (stmt_info->stmt), i);
8656 67747 : for (unsigned j = 0; j < vec_oprnds.length (); j++)
8657 36099 : add_phi_arg (new_phis[j], vec_oprnds[j], e, UNKNOWN_LOCATION);
8658 31648 : }
8659 : /* We should have at least one already vectorized child. */
8660 12757 : gcc_assert (new_phis.exists ());
8661 :
8662 12757 : return true;
8663 12757 : }
8664 :
8665 : /* Vectorizes first order recurrences. An overview of the transformation
8666 : is described below. Suppose we have the following loop.
8667 :
8668 : int t = 0;
8669 : for (int i = 0; i < n; ++i)
8670 : {
8671 : b[i] = a[i] - t;
8672 : t = a[i];
8673 : }
8674 :
8675 : There is a first-order recurrence on 'a'. For this loop, the scalar IR
8676 : looks (simplified) like:
8677 :
8678 : scalar.preheader:
8679 : init = 0;
8680 :
8681 : scalar.body:
8682 : i = PHI <0(scalar.preheader), i+1(scalar.body)>
8683 : _2 = PHI <(init(scalar.preheader), <_1(scalar.body)>
8684 : _1 = a[i]
8685 : b[i] = _1 - _2
8686 : if (i < n) goto scalar.body
8687 :
8688 : In this example, _2 is a recurrence because it's value depends on the
8689 : previous iteration. We vectorize this as (VF = 4)
8690 :
8691 : vector.preheader:
8692 : vect_init = vect_cst(..., ..., ..., 0)
8693 :
8694 : vector.body
8695 : i = PHI <0(vector.preheader), i+4(vector.body)>
8696 : vect_1 = PHI <vect_init(vector.preheader), v2(vector.body)>
8697 : vect_2 = a[i, i+1, i+2, i+3];
8698 : vect_3 = vec_perm (vect_1, vect_2, { 3, 4, 5, 6 })
8699 : b[i, i+1, i+2, i+3] = vect_2 - vect_3
8700 : if (..) goto vector.body
8701 :
8702 : In this function, vectorizable_recurr, we code generate both the
8703 : vector PHI node and the permute since those together compute the
8704 : vectorized value of the scalar PHI. We do not yet have the
8705 : backedge value to fill in there nor into the vec_perm. Those
8706 : are filled in vect_schedule_scc.
8707 :
8708 : TODO: Since the scalar loop does not have a use of the recurrence
8709 : outside of the loop the natural way to implement peeling via
8710 : vectorizing the live value doesn't work. For now peeling of loops
8711 : with a recurrence is not implemented. For SLP the supported cases
8712 : are restricted to those requiring a single vector recurrence PHI. */
8713 :
8714 : bool
8715 198240 : vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
8716 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
8717 : {
8718 198240 : if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
8719 : return false;
8720 :
8721 38885 : gphi *phi = as_a<gphi *> (stmt_info->stmt);
8722 :
8723 : /* So far we only support first-order recurrence auto-vectorization. */
8724 38885 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
8725 : return false;
8726 :
8727 424 : tree vectype = SLP_TREE_VECTYPE (slp_node);
8728 424 : unsigned ncopies = vect_get_num_copies (loop_vinfo, slp_node);
8729 424 : poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8730 424 : unsigned dist = SLP_TREE_LANES (slp_node);
8731 : /* We need to be able to make progress with a single vector. */
8732 424 : if (maybe_gt (dist * 2, nunits))
8733 : {
8734 0 : if (dump_enabled_p ())
8735 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8736 : "first order recurrence exceeds half of "
8737 : "a vector\n");
8738 : return false;
8739 : }
8740 :
8741 : /* We need to be able to build a { ..., a, b } init vector with
8742 : dist number of distinct trailing values. Always possible
8743 : when dist == 1 or when nunits is constant or when the initializations
8744 : are uniform. */
8745 424 : tree uniform_initval = NULL_TREE;
8746 424 : edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
8747 1720 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8748 : {
8749 460 : gphi *phi = as_a <gphi *> (s->stmt);
8750 460 : if (! uniform_initval)
8751 424 : uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8752 36 : else if (! operand_equal_p (uniform_initval,
8753 36 : PHI_ARG_DEF_FROM_EDGE (phi, pe)))
8754 : {
8755 : uniform_initval = NULL_TREE;
8756 : break;
8757 : }
8758 : }
8759 424 : if (!uniform_initval && !nunits.is_constant ())
8760 : {
8761 : if (dump_enabled_p ())
8762 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8763 : "cannot build initialization vector for "
8764 : "first order recurrence\n");
8765 : return false;
8766 : }
8767 :
8768 : /* First-order recurrence autovectorization needs to handle permutation
8769 : with indices = [nunits-1, nunits, nunits+1, ...]. */
8770 424 : vec_perm_builder sel (nunits, 1, 3);
8771 1696 : for (int i = 0; i < 3; ++i)
8772 1272 : sel.quick_push (nunits - dist + i);
8773 424 : vec_perm_indices indices (sel, 2, nunits);
8774 :
8775 424 : if (cost_vec) /* transformation not required. */
8776 : {
8777 381 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
8778 : indices))
8779 : return false;
8780 :
8781 : /* We eventually need to set a vector type on invariant
8782 : arguments. */
8783 : unsigned j;
8784 : slp_tree child;
8785 807 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
8786 538 : if (!vect_maybe_update_slp_op_vectype (child, vectype))
8787 : {
8788 0 : if (dump_enabled_p ())
8789 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8790 : "incompatible vector types for "
8791 : "invariants\n");
8792 : return false;
8793 : }
8794 :
8795 : /* Verify we have set up compatible types. */
8796 269 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8797 269 : slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
8798 269 : tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
8799 269 : if (!types_compatible_p (latch_vectype, vectype))
8800 : return false;
8801 :
8802 : /* The recurrence costs the initialization vector and one permute
8803 : for each copy. With SLP the prologue value is explicitly
8804 : represented and costed separately. */
8805 269 : unsigned prologue_cost = 0;
8806 269 : unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
8807 : slp_node, 0, vect_body);
8808 269 : if (dump_enabled_p ())
8809 51 : dump_printf_loc (MSG_NOTE, vect_location,
8810 : "vectorizable_recurr: inside_cost = %d, "
8811 : "prologue_cost = %d .\n", inside_cost,
8812 : prologue_cost);
8813 :
8814 269 : SLP_TREE_TYPE (slp_node) = recurr_info_type;
8815 269 : return true;
8816 : }
8817 :
8818 43 : tree vec_init;
8819 43 : if (! uniform_initval)
8820 : {
8821 6 : vec<constructor_elt, va_gc> *v = NULL;
8822 6 : vec_alloc (v, nunits.to_constant ());
8823 39 : for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
8824 27 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
8825 : build_zero_cst (TREE_TYPE (vectype)));
8826 39 : for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
8827 : {
8828 21 : gphi *phi = as_a <gphi *> (s->stmt);
8829 21 : tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
8830 21 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
8831 21 : TREE_TYPE (preheader)))
8832 : {
8833 0 : gimple_seq stmts = NULL;
8834 0 : preheader = gimple_convert (&stmts,
8835 0 : TREE_TYPE (vectype), preheader);
8836 0 : gsi_insert_seq_on_edge_immediate (pe, stmts);
8837 : }
8838 21 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
8839 : }
8840 6 : vec_init = build_constructor (vectype, v);
8841 : }
8842 : else
8843 : vec_init = uniform_initval;
8844 43 : vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
8845 :
8846 : /* Create the vectorized first-order PHI node. */
8847 43 : tree vec_dest = vect_get_new_vect_var (vectype,
8848 : vect_simple_var, "vec_recur_");
8849 43 : basic_block bb = gimple_bb (phi);
8850 43 : gphi *new_phi = create_phi_node (vec_dest, bb);
8851 43 : add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
8852 :
8853 : /* Insert shuffles the first-order recurrence autovectorization.
8854 : result = VEC_PERM <vec_recur, vect_1, index[nunits-1, nunits, ...]>. */
8855 43 : tree perm = vect_gen_perm_mask_checked (vectype, indices);
8856 :
8857 : /* Insert the required permute after the latch definition. The
8858 : second and later operands are tentative and will be updated when we have
8859 : vectorized the latch definition. */
8860 43 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
8861 43 : gimple *latch_def = SSA_NAME_DEF_STMT (PHI_ARG_DEF_FROM_EDGE (phi, le));
8862 43 : gimple_stmt_iterator gsi2 = gsi_for_stmt (latch_def);
8863 49 : do
8864 : {
8865 49 : gsi_next (&gsi2);
8866 : }
8867 : /* Skip inserted vectorized stmts for the latch definition. We have to
8868 : insert after those. */
8869 92 : while (gsi_stmt (gsi2) && gimple_uid (gsi_stmt (gsi2)) == 0);
8870 :
8871 123 : for (unsigned i = 0; i < ncopies; ++i)
8872 : {
8873 80 : vec_dest = make_ssa_name (vectype);
8874 80 : gassign *vperm
8875 123 : = gimple_build_assign (vec_dest, VEC_PERM_EXPR,
8876 43 : i == 0 ? gimple_phi_result (new_phi) : NULL,
8877 : NULL, perm);
8878 80 : vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
8879 :
8880 80 : slp_node->push_vec_def (vperm);
8881 : }
8882 :
8883 : return true;
8884 424 : }
8885 :
8886 : /* Return true if VECTYPE represents a vector that requires lowering
8887 : by the vector lowering pass. */
8888 :
8889 : bool
8890 681096 : vect_emulated_vector_p (tree vectype)
8891 : {
8892 1362192 : return (!VECTOR_MODE_P (TYPE_MODE (vectype))
8893 685226 : && (!VECTOR_BOOLEAN_TYPE_P (vectype)
8894 4112 : || TYPE_PRECISION (TREE_TYPE (vectype)) != 1));
8895 : }
8896 :
8897 : /* Return true if we can emulate CODE on an integer mode representation
8898 : of a vector. */
8899 :
8900 : bool
8901 12238 : vect_can_vectorize_without_simd_p (tree_code code)
8902 : {
8903 12238 : switch (code)
8904 : {
8905 : case PLUS_EXPR:
8906 : case MINUS_EXPR:
8907 : case NEGATE_EXPR:
8908 : case BIT_AND_EXPR:
8909 : case BIT_IOR_EXPR:
8910 : case BIT_XOR_EXPR:
8911 : case BIT_NOT_EXPR:
8912 : return true;
8913 :
8914 11626 : default:
8915 11626 : return false;
8916 : }
8917 : }
8918 :
8919 : /* Likewise, but taking a code_helper. */
8920 :
8921 : bool
8922 1003 : vect_can_vectorize_without_simd_p (code_helper code)
8923 : {
8924 1003 : return (code.is_tree_code ()
8925 1003 : && vect_can_vectorize_without_simd_p (tree_code (code)));
8926 : }
8927 :
8928 : /* Create vector init for vectorized iv. */
8929 : static tree
8930 929 : vect_create_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
8931 : tree step_expr, poly_uint64 nunits,
8932 : tree vectype,
8933 : enum vect_induction_op_type induction_type)
8934 : {
8935 929 : unsigned HOST_WIDE_INT const_nunits;
8936 929 : tree vec_shift, vec_init, new_name;
8937 929 : unsigned i;
8938 929 : tree itype = TREE_TYPE (vectype);
8939 :
8940 : /* iv_loop is the loop to be vectorized. Create:
8941 : vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr). */
8942 929 : new_name = gimple_convert (stmts, itype, init_expr);
8943 929 : switch (induction_type)
8944 : {
8945 21 : case vect_step_op_shr:
8946 21 : case vect_step_op_shl:
8947 : /* Build the Initial value from shift_expr. */
8948 21 : vec_init = gimple_build_vector_from_val (stmts,
8949 : vectype,
8950 : new_name);
8951 21 : vec_shift = gimple_build (stmts, VEC_SERIES_EXPR, vectype,
8952 : build_zero_cst (itype), step_expr);
8953 21 : vec_init = gimple_build (stmts,
8954 : (induction_type == vect_step_op_shr
8955 : ? RSHIFT_EXPR : LSHIFT_EXPR),
8956 : vectype, vec_init, vec_shift);
8957 21 : break;
8958 :
8959 833 : case vect_step_op_neg:
8960 833 : {
8961 833 : vec_init = gimple_build_vector_from_val (stmts,
8962 : vectype,
8963 : new_name);
8964 833 : tree vec_neg = gimple_build (stmts, NEGATE_EXPR,
8965 : vectype, vec_init);
8966 : /* The encoding has 2 interleaved stepped patterns. */
8967 833 : vec_perm_builder sel (nunits, 2, 3);
8968 833 : sel.quick_grow (6);
8969 4165 : for (i = 0; i < 3; i++)
8970 : {
8971 2499 : sel[2 * i] = i;
8972 2499 : sel[2 * i + 1] = i + nunits;
8973 : }
8974 833 : vec_perm_indices indices (sel, 2, nunits);
8975 : /* Don't use vect_gen_perm_mask_checked since can_vec_perm_const_p may
8976 : fail when vec_init is const vector. In that situation vec_perm is not
8977 : really needed. */
8978 833 : tree perm_mask_even
8979 833 : = vect_gen_perm_mask_any (vectype, indices);
8980 833 : vec_init = gimple_build (stmts, VEC_PERM_EXPR,
8981 : vectype,
8982 : vec_init, vec_neg,
8983 : perm_mask_even);
8984 833 : }
8985 833 : break;
8986 :
8987 75 : case vect_step_op_mul:
8988 75 : {
8989 : /* Use unsigned mult to avoid UD integer overflow. */
8990 75 : gcc_assert (nunits.is_constant (&const_nunits));
8991 75 : tree utype = unsigned_type_for (itype);
8992 75 : tree uvectype = build_vector_type (utype,
8993 75 : TYPE_VECTOR_SUBPARTS (vectype));
8994 75 : new_name = gimple_convert (stmts, utype, new_name);
8995 75 : vec_init = gimple_build_vector_from_val (stmts,
8996 : uvectype,
8997 : new_name);
8998 75 : tree_vector_builder elts (uvectype, const_nunits, 1);
8999 75 : tree elt_step = build_one_cst (utype);
9000 :
9001 75 : elts.quick_push (elt_step);
9002 655 : for (i = 1; i < const_nunits; i++)
9003 : {
9004 : /* Create: new_name_i = new_name + step_expr. */
9005 505 : elt_step = gimple_build (stmts, MULT_EXPR,
9006 : utype, elt_step, step_expr);
9007 505 : elts.quick_push (elt_step);
9008 : }
9009 : /* Create a vector from [new_name_0, new_name_1, ...,
9010 : new_name_nunits-1]. */
9011 75 : tree vec_mul = gimple_build_vector (stmts, &elts);
9012 75 : vec_init = gimple_build (stmts, MULT_EXPR, uvectype,
9013 : vec_init, vec_mul);
9014 75 : vec_init = gimple_convert (stmts, vectype, vec_init);
9015 75 : }
9016 75 : break;
9017 :
9018 0 : default:
9019 0 : gcc_unreachable ();
9020 : }
9021 :
9022 929 : return vec_init;
9023 : }
9024 :
9025 : /* Peel init_expr by skip_niter for induction_type. */
9026 : tree
9027 83 : vect_peel_nonlinear_iv_init (gimple_seq* stmts, tree init_expr,
9028 : tree skip_niters, tree step_expr,
9029 : enum vect_induction_op_type induction_type,
9030 : bool early_exit_p)
9031 : {
9032 83 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST || early_exit_p);
9033 83 : tree type = TREE_TYPE (init_expr);
9034 83 : unsigned prec = TYPE_PRECISION (type);
9035 83 : switch (induction_type)
9036 : {
9037 : /* neg inductions are typically not used for loop termination conditions but
9038 : are typically implemented as b = -b. That is every scalar iteration b is
9039 : negated. That means that for the initial value of b we will have to
9040 : determine whether the number of skipped iteration is a multiple of 2
9041 : because every 2 scalar iterations we are back at "b". */
9042 0 : case vect_step_op_neg:
9043 : /* For early exits the neg induction will always be the same value at the
9044 : start of the iteration. */
9045 0 : if (early_exit_p)
9046 : break;
9047 :
9048 0 : if (TREE_INT_CST_LOW (skip_niters) % 2)
9049 0 : init_expr = gimple_build (stmts, NEGATE_EXPR, type, init_expr);
9050 : /* else no change. */
9051 : break;
9052 :
9053 12 : case vect_step_op_shr:
9054 12 : case vect_step_op_shl:
9055 12 : skip_niters = fold_build1 (NOP_EXPR, type, skip_niters);
9056 12 : step_expr = fold_build1 (NOP_EXPR, type, step_expr);
9057 12 : step_expr = fold_build2 (MULT_EXPR, type, step_expr, skip_niters);
9058 : /* When shift mount >= precision, need to avoid UD.
9059 : In the original loop, there's no UD, and according to semantic,
9060 : init_expr should be 0 for lshr, ashl, and >>= (prec - 1) for ashr. */
9061 12 : if ((!tree_fits_uhwi_p (step_expr)
9062 12 : || tree_to_uhwi (step_expr) >= prec)
9063 6 : && !early_exit_p)
9064 : {
9065 6 : if (induction_type == vect_step_op_shl
9066 6 : || TYPE_UNSIGNED (type))
9067 4 : init_expr = build_zero_cst (type);
9068 : else
9069 2 : init_expr = gimple_build (stmts, RSHIFT_EXPR, type,
9070 : init_expr,
9071 4 : wide_int_to_tree (type, prec - 1));
9072 : }
9073 : else
9074 : {
9075 8 : init_expr = fold_build2 ((induction_type == vect_step_op_shr
9076 : ? RSHIFT_EXPR : LSHIFT_EXPR),
9077 : type, init_expr, step_expr);
9078 6 : init_expr = force_gimple_operand (init_expr, stmts, false, NULL);
9079 : }
9080 : break;
9081 :
9082 71 : case vect_step_op_mul:
9083 71 : {
9084 : /* Due to UB we can't support vect_step_op_mul with early break for now.
9085 : so assert and block. */
9086 71 : gcc_assert (TREE_CODE (skip_niters) == INTEGER_CST);
9087 71 : tree utype = unsigned_type_for (type);
9088 71 : init_expr = gimple_convert (stmts, utype, init_expr);
9089 71 : wide_int skipn = wi::to_wide (skip_niters);
9090 71 : wide_int begin = wi::to_wide (step_expr);
9091 71 : auto_mpz base, exp, mod, res;
9092 71 : wi::to_mpz (begin, base, TYPE_SIGN (type));
9093 71 : wi::to_mpz (skipn, exp, UNSIGNED);
9094 71 : mpz_ui_pow_ui (mod, 2, TYPE_PRECISION (type));
9095 71 : mpz_powm (res, base, exp, mod);
9096 71 : begin = wi::from_mpz (utype, res, true);
9097 71 : tree mult_expr = wide_int_to_tree (utype, begin);
9098 71 : init_expr = gimple_build (stmts, MULT_EXPR, utype,
9099 : init_expr, mult_expr);
9100 71 : init_expr = gimple_convert (stmts, type, init_expr);
9101 71 : }
9102 71 : break;
9103 :
9104 0 : default:
9105 0 : gcc_unreachable ();
9106 : }
9107 :
9108 83 : return init_expr;
9109 : }
9110 :
9111 : /* Create vector step for vectorized iv. */
9112 : static tree
9113 1218 : vect_create_nonlinear_iv_step (gimple_seq* stmts, tree step_expr,
9114 : poly_uint64 vf,
9115 : enum vect_induction_op_type induction_type)
9116 : {
9117 1218 : tree expr = build_int_cst (TREE_TYPE (step_expr), vf);
9118 1218 : tree new_name = NULL;
9119 : /* Step should be pow (step, vf) for mult induction. */
9120 1218 : if (induction_type == vect_step_op_mul)
9121 : {
9122 75 : gcc_assert (vf.is_constant ());
9123 75 : wide_int begin = wi::to_wide (step_expr);
9124 :
9125 580 : for (unsigned i = 0; i != vf.to_constant () - 1; i++)
9126 505 : begin = wi::mul (begin, wi::to_wide (step_expr));
9127 :
9128 75 : new_name = wide_int_to_tree (TREE_TYPE (step_expr), begin);
9129 75 : }
9130 1143 : else if (induction_type == vect_step_op_neg)
9131 : /* Do nothing. */
9132 : ;
9133 : else
9134 24 : new_name = gimple_build (stmts, MULT_EXPR, TREE_TYPE (step_expr),
9135 : expr, step_expr);
9136 1218 : return new_name;
9137 : }
9138 :
9139 : static tree
9140 1218 : vect_create_nonlinear_iv_vec_step (loop_vec_info loop_vinfo,
9141 : stmt_vec_info stmt_info,
9142 : tree new_name, tree vectype,
9143 : enum vect_induction_op_type induction_type)
9144 : {
9145 : /* No step is needed for neg induction. */
9146 1218 : if (induction_type == vect_step_op_neg)
9147 : return NULL;
9148 :
9149 99 : tree t = unshare_expr (new_name);
9150 99 : gcc_assert (CONSTANT_CLASS_P (new_name)
9151 : || TREE_CODE (new_name) == SSA_NAME);
9152 99 : tree new_vec = build_vector_from_val (vectype, t);
9153 99 : tree vec_step = vect_init_vector (loop_vinfo, stmt_info,
9154 : new_vec, vectype, NULL);
9155 99 : return vec_step;
9156 : }
9157 :
9158 : /* Update vectorized iv with vect_step, induc_def is init. */
9159 : static tree
9160 1424 : vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
9161 : tree induc_def, tree vec_step,
9162 : enum vect_induction_op_type induction_type)
9163 : {
9164 1424 : tree vec_def = induc_def;
9165 1424 : switch (induction_type)
9166 : {
9167 75 : case vect_step_op_mul:
9168 75 : {
9169 : /* Use unsigned mult to avoid UD integer overflow. */
9170 75 : tree uvectype = unsigned_type_for (vectype);
9171 75 : vec_def = gimple_convert (stmts, uvectype, vec_def);
9172 75 : vec_step = gimple_convert (stmts, uvectype, vec_step);
9173 75 : vec_def = gimple_build (stmts, MULT_EXPR, uvectype,
9174 : vec_def, vec_step);
9175 75 : vec_def = gimple_convert (stmts, vectype, vec_def);
9176 : }
9177 75 : break;
9178 :
9179 36 : case vect_step_op_shr:
9180 36 : vec_def = gimple_build (stmts, RSHIFT_EXPR, vectype,
9181 : vec_def, vec_step);
9182 36 : break;
9183 :
9184 6 : case vect_step_op_shl:
9185 6 : vec_def = gimple_build (stmts, LSHIFT_EXPR, vectype,
9186 : vec_def, vec_step);
9187 6 : break;
9188 : case vect_step_op_neg:
9189 : vec_def = induc_def;
9190 : /* Do nothing. */
9191 : break;
9192 0 : default:
9193 0 : gcc_unreachable ();
9194 : }
9195 :
9196 1424 : return vec_def;
9197 :
9198 : }
9199 :
9200 : /* Function vectorizable_nonlinear_induction
9201 :
9202 : Check if STMT_INFO performs an nonlinear induction computation that can be
9203 : vectorized. If VEC_STMT is also passed, vectorize the induction PHI: create
9204 : a vectorized phi to replace it, put it in VEC_STMT, and add it to the same
9205 : basic block.
9206 : Return true if STMT_INFO is vectorizable in this way. */
9207 :
9208 : static bool
9209 9685 : vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
9210 : stmt_vec_info stmt_info,
9211 : slp_tree slp_node,
9212 : stmt_vector_for_cost *cost_vec)
9213 : {
9214 9685 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9215 9685 : unsigned ncopies;
9216 9685 : bool nested_in_vect_loop = false;
9217 9685 : class loop *iv_loop;
9218 9685 : tree vec_def;
9219 9685 : edge pe = loop_preheader_edge (loop);
9220 9685 : basic_block new_bb;
9221 9685 : tree vec_init, vec_step;
9222 9685 : tree new_name;
9223 9685 : gimple *new_stmt;
9224 9685 : gphi *induction_phi;
9225 9685 : tree induc_def, vec_dest;
9226 9685 : tree init_expr, step_expr;
9227 9685 : tree niters_skip;
9228 9685 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9229 9685 : unsigned i;
9230 9685 : gimple_stmt_iterator si;
9231 :
9232 9685 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9233 :
9234 9685 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9235 9685 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9236 9685 : enum vect_induction_op_type induction_type
9237 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9238 :
9239 9685 : gcc_assert (induction_type > vect_step_op_add);
9240 :
9241 9685 : ncopies = vect_get_num_copies (loop_vinfo, slp_node);
9242 9685 : gcc_assert (ncopies >= 1);
9243 :
9244 : /* FORNOW. Only handle nonlinear induction in the same loop. */
9245 9685 : if (nested_in_vect_loop_p (loop, stmt_info))
9246 : {
9247 0 : if (dump_enabled_p ())
9248 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9249 : "nonlinear induction in nested loop.\n");
9250 : return false;
9251 : }
9252 :
9253 9685 : iv_loop = loop;
9254 9685 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9255 :
9256 : /* TODO: Support multi-lane SLP for nonlinear iv. There should be separate
9257 : vector iv update for each iv and a permutation to generate wanted
9258 : vector iv. */
9259 9685 : if (SLP_TREE_LANES (slp_node) > 1)
9260 : {
9261 0 : if (dump_enabled_p ())
9262 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9263 : "SLP induction not supported for nonlinear"
9264 : " induction.\n");
9265 : return false;
9266 : }
9267 :
9268 9685 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
9269 : {
9270 0 : if (dump_enabled_p ())
9271 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9272 : "floating point nonlinear induction vectorization"
9273 : " not supported.\n");
9274 : return false;
9275 : }
9276 :
9277 9685 : step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9278 9685 : init_expr = vect_phi_initial_value (phi);
9279 9685 : gcc_assert (step_expr != NULL_TREE && init_expr != NULL
9280 : && TREE_CODE (step_expr) == INTEGER_CST);
9281 : /* step_expr should be aligned with init_expr,
9282 : .i.e. uint64 a >> 1, step is int, but vector<uint64> shift is used. */
9283 9685 : step_expr = fold_convert (TREE_TYPE (vectype), step_expr);
9284 :
9285 9685 : if (TREE_CODE (init_expr) == INTEGER_CST)
9286 4111 : init_expr = fold_convert (TREE_TYPE (vectype), init_expr);
9287 5574 : else if (!tree_nop_conversion_p (TREE_TYPE (vectype), TREE_TYPE (init_expr)))
9288 : {
9289 : /* INIT_EXPR could be a bit_field, bail out for such case. */
9290 4 : if (dump_enabled_p ())
9291 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9292 : "nonlinear induction vectorization failed:"
9293 : " component type of vectype is not a nop conversion"
9294 : " from type of init_expr.\n");
9295 : return false;
9296 : }
9297 :
9298 9681 : switch (induction_type)
9299 : {
9300 3743 : case vect_step_op_neg:
9301 3743 : if (maybe_eq (TYPE_VECTOR_SUBPARTS (vectype), 1u))
9302 : return false;
9303 3581 : if (TREE_CODE (init_expr) != INTEGER_CST
9304 285 : && TREE_CODE (init_expr) != REAL_CST)
9305 : {
9306 : /* Check for backend support of NEGATE_EXPR and vec_perm. */
9307 285 : if (!directly_supported_p (NEGATE_EXPR, vectype))
9308 0 : return false;
9309 :
9310 : /* The encoding has 2 interleaved stepped patterns. */
9311 285 : vec_perm_builder sel (nunits, 2, 3);
9312 285 : machine_mode mode = TYPE_MODE (vectype);
9313 285 : sel.quick_grow (6);
9314 1425 : for (i = 0; i < 3; i++)
9315 : {
9316 855 : sel[i * 2] = i;
9317 855 : sel[i * 2 + 1] = i + nunits;
9318 : }
9319 285 : vec_perm_indices indices (sel, 2, nunits);
9320 285 : if (!can_vec_perm_const_p (mode, mode, indices))
9321 0 : return false;
9322 285 : }
9323 : break;
9324 :
9325 1072 : case vect_step_op_mul:
9326 1072 : {
9327 : /* Check for backend support of MULT_EXPR. */
9328 1072 : if (!directly_supported_p (MULT_EXPR, vectype))
9329 : return false;
9330 :
9331 : /* ?? How to construct vector step for variable number vector.
9332 : [ 1, step, pow (step, 2), pow (step, 4), .. ]. */
9333 : if (!vf.is_constant ())
9334 : return false;
9335 : }
9336 : break;
9337 :
9338 4512 : case vect_step_op_shr:
9339 : /* Check for backend support of RSHIFT_EXPR. */
9340 4512 : if (!directly_supported_p (RSHIFT_EXPR, vectype, optab_vector))
9341 : return false;
9342 :
9343 : /* Don't shift more than type precision to avoid UB. */
9344 38 : if (!tree_fits_uhwi_p (step_expr)
9345 38 : || maybe_ge (vf * tree_to_uhwi (step_expr),
9346 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9347 : return false;
9348 : break;
9349 :
9350 354 : case vect_step_op_shl:
9351 : /* Check for backend support of RSHIFT_EXPR. */
9352 354 : if (!directly_supported_p (LSHIFT_EXPR, vectype, optab_vector))
9353 : return false;
9354 :
9355 : /* Don't shift more than type precision to avoid UB. */
9356 12 : if (!tree_fits_uhwi_p (step_expr)
9357 12 : || maybe_ge (vf * tree_to_uhwi (step_expr),
9358 : TYPE_PRECISION (TREE_TYPE (init_expr))))
9359 : return false;
9360 : break;
9361 :
9362 0 : default:
9363 0 : gcc_unreachable ();
9364 : }
9365 :
9366 4461 : if (cost_vec) /* transformation not required. */
9367 : {
9368 3532 : unsigned inside_cost = 0, prologue_cost = 0;
9369 : /* loop cost for vec_loop. Neg induction doesn't have any
9370 : inside_cost. */
9371 3532 : inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
9372 : slp_node, 0, vect_body);
9373 :
9374 : /* loop cost for vec_loop. Neg induction doesn't have any
9375 : inside_cost. */
9376 3532 : if (induction_type == vect_step_op_neg)
9377 2748 : inside_cost = 0;
9378 :
9379 : /* prologue cost for vec_init and vec_step. */
9380 3532 : prologue_cost = record_stmt_cost (cost_vec, 2, scalar_to_vec,
9381 : slp_node, 0, vect_prologue);
9382 :
9383 3532 : if (dump_enabled_p ())
9384 76 : dump_printf_loc (MSG_NOTE, vect_location,
9385 : "vect_model_induction_cost: inside_cost = %d, "
9386 : "prologue_cost = %d. \n", inside_cost,
9387 : prologue_cost);
9388 :
9389 3532 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9390 3532 : DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
9391 3532 : return true;
9392 : }
9393 :
9394 : /* Transform. */
9395 :
9396 : /* Compute a vector variable, initialized with the first VF values of
9397 : the induction variable. E.g., for an iv with IV_PHI='X' and
9398 : evolution S, for a vector of 4 units, we want to compute:
9399 : [X, X + S, X + 2*S, X + 3*S]. */
9400 :
9401 929 : if (dump_enabled_p ())
9402 40 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9403 :
9404 929 : pe = loop_preheader_edge (iv_loop);
9405 : /* Find the first insertion point in the BB. */
9406 929 : basic_block bb = gimple_bb (phi);
9407 929 : si = gsi_after_labels (bb);
9408 :
9409 929 : gimple_seq stmts = NULL;
9410 :
9411 929 : niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
9412 : /* If we are using the loop mask to "peel" for alignment then we need
9413 : to adjust the start value here. */
9414 929 : if (niters_skip != NULL_TREE)
9415 0 : init_expr = vect_peel_nonlinear_iv_init (&stmts, init_expr, niters_skip,
9416 : step_expr, induction_type, false);
9417 :
9418 929 : vec_init = vect_create_nonlinear_iv_init (&stmts, init_expr,
9419 : step_expr, nunits, vectype,
9420 : induction_type);
9421 929 : if (stmts)
9422 : {
9423 167 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9424 167 : gcc_assert (!new_bb);
9425 : }
9426 :
9427 929 : stmts = NULL;
9428 929 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9429 : vf, induction_type);
9430 929 : if (stmts)
9431 : {
9432 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
9433 0 : gcc_assert (!new_bb);
9434 : }
9435 :
9436 929 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9437 : new_name, vectype,
9438 : induction_type);
9439 : /* Create the following def-use cycle:
9440 : loop prolog:
9441 : vec_init = ...
9442 : vec_step = ...
9443 : loop:
9444 : vec_iv = PHI <vec_init, vec_loop>
9445 : ...
9446 : STMT
9447 : ...
9448 : vec_loop = vec_iv + vec_step; */
9449 :
9450 : /* Create the induction-phi that defines the induction-operand. */
9451 929 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
9452 929 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
9453 929 : induc_def = PHI_RESULT (induction_phi);
9454 :
9455 : /* Create the iv update inside the loop. */
9456 929 : stmts = NULL;
9457 929 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9458 : induc_def, vec_step,
9459 : induction_type);
9460 :
9461 929 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9462 929 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9463 :
9464 : /* Set the arguments of the phi node: */
9465 929 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
9466 929 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
9467 : UNKNOWN_LOCATION);
9468 :
9469 929 : slp_node->push_vec_def (induction_phi);
9470 :
9471 : /* In case that vectorization factor (VF) is bigger than the number
9472 : of elements that we can fit in a vectype (nunits), we have to generate
9473 : more than one vector stmt - i.e - we need to "unroll" the
9474 : vector stmt by a factor VF/nunits. For more details see documentation
9475 : in vectorizable_operation. */
9476 :
9477 929 : if (ncopies > 1)
9478 : {
9479 289 : stmts = NULL;
9480 : /* FORNOW. This restriction should be relaxed. */
9481 289 : gcc_assert (!nested_in_vect_loop);
9482 :
9483 289 : new_name = vect_create_nonlinear_iv_step (&stmts, step_expr,
9484 : nunits, induction_type);
9485 :
9486 289 : vec_step = vect_create_nonlinear_iv_vec_step (loop_vinfo, stmt_info,
9487 : new_name, vectype,
9488 : induction_type);
9489 289 : vec_def = induc_def;
9490 1073 : for (i = 1; i < ncopies; i++)
9491 : {
9492 : /* vec_i = vec_prev + vec_step. */
9493 495 : stmts = NULL;
9494 495 : vec_def = vect_update_nonlinear_iv (&stmts, vectype,
9495 : vec_def, vec_step,
9496 : induction_type);
9497 495 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
9498 495 : new_stmt = SSA_NAME_DEF_STMT (vec_def);
9499 495 : slp_node->push_vec_def (new_stmt);
9500 : }
9501 : }
9502 :
9503 929 : if (dump_enabled_p ())
9504 80 : dump_printf_loc (MSG_NOTE, vect_location,
9505 : "transform induction: created def-use cycle: %G%G",
9506 40 : (gimple *) induction_phi, SSA_NAME_DEF_STMT (vec_def));
9507 :
9508 : return true;
9509 : }
9510 :
9511 : /* Return true if the scalar initial values and steps of the SLP induction
9512 : lanes allow the first CANDIDATE_NIVS IVs to be reused circularly for the
9513 : remaining lanes. */
9514 : static bool
9515 26 : vect_slp_induction_reuse_p (tree *steps, tree *inits, unsigned group_size,
9516 : unsigned HOST_WIDE_INT const_nunits,
9517 : unsigned candidate_nivs, unsigned nivs)
9518 : {
9519 26 : gcc_assert (candidate_nivs > 0);
9520 26 : gcc_assert (candidate_nivs < nivs);
9521 :
9522 : /* This function compares only STEPS and INITS, so all checked lanes must
9523 : precede the first wrap of the SLP group. */
9524 26 : gcc_assert (nivs * const_nunits <= group_size);
9525 :
9526 62 : for (unsigned ivn = candidate_nivs; ivn < nivs; ++ivn)
9527 : {
9528 42 : unsigned reuse_ivn = ivn % candidate_nivs;
9529 172 : for (unsigned HOST_WIDE_INT eltn = 0; eltn < const_nunits; ++eltn)
9530 : {
9531 136 : unsigned HOST_WIDE_INT elt = ivn * const_nunits + eltn;
9532 136 : unsigned HOST_WIDE_INT reused_elt
9533 136 : = reuse_ivn * const_nunits + eltn;
9534 :
9535 136 : if (!operand_equal_p (steps[elt], steps[reused_elt], 0)
9536 136 : || !operand_equal_p (inits[elt], inits[reused_elt], 0))
9537 : return false;
9538 : }
9539 : }
9540 :
9541 : return true;
9542 : }
9543 :
9544 : /* Function vectorizable_induction
9545 :
9546 : Check if STMT_INFO performs an induction computation that can be vectorized.
9547 : If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
9548 : phi to replace it, put it in VEC_STMT, and add it to the same basic block.
9549 : Return true if STMT_INFO is vectorizable in this way. */
9550 :
9551 : bool
9552 341856 : vectorizable_induction (loop_vec_info loop_vinfo,
9553 : stmt_vec_info stmt_info,
9554 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
9555 : {
9556 341856 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
9557 341856 : bool nested_in_vect_loop = false;
9558 341856 : class loop *iv_loop;
9559 341856 : tree vec_def;
9560 341856 : edge pe = loop_preheader_edge (loop);
9561 341856 : basic_block new_bb;
9562 341856 : tree vec_init = NULL_TREE, vec_step, t;
9563 341856 : tree new_name;
9564 341856 : gphi *induction_phi;
9565 341856 : tree induc_def, vec_dest;
9566 341856 : unsigned i;
9567 341856 : tree index_vectype = NULL_TREE;
9568 341856 : gimple_stmt_iterator si;
9569 341856 : enum vect_induction_op_type induction_type
9570 : = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info);
9571 :
9572 341856 : gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
9573 182501 : if (!phi)
9574 : return false;
9575 :
9576 182501 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
9577 : return false;
9578 :
9579 : /* Make sure it was recognized as induction computation. */
9580 182501 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
9581 : return false;
9582 :
9583 : /* Handle nonlinear induction in a separate place. */
9584 178393 : if (induction_type != vect_step_op_add)
9585 9685 : return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
9586 9685 : slp_node, cost_vec);
9587 :
9588 168708 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9589 168708 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9590 :
9591 : /* FORNOW. These restrictions should be relaxed. */
9592 168708 : if (nested_in_vect_loop_p (loop, stmt_info))
9593 : {
9594 776 : imm_use_iterator imm_iter;
9595 776 : use_operand_p use_p;
9596 776 : gimple *exit_phi;
9597 776 : edge latch_e;
9598 776 : tree loop_arg;
9599 :
9600 776 : exit_phi = NULL;
9601 776 : latch_e = loop_latch_edge (loop->inner);
9602 776 : loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
9603 1586 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
9604 : {
9605 836 : gimple *use_stmt = USE_STMT (use_p);
9606 836 : if (is_gimple_debug (use_stmt))
9607 36 : continue;
9608 :
9609 800 : if (!flow_bb_inside_loop_p (loop->inner, gimple_bb (use_stmt)))
9610 : {
9611 : exit_phi = use_stmt;
9612 : break;
9613 : }
9614 776 : }
9615 776 : if (exit_phi)
9616 : {
9617 26 : stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi);
9618 26 : if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
9619 10 : && !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
9620 : {
9621 16 : if (dump_enabled_p ())
9622 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9623 : "inner-loop induction only used outside "
9624 : "of the outer vectorized loop.\n");
9625 16 : return false;
9626 : }
9627 : }
9628 :
9629 760 : nested_in_vect_loop = true;
9630 760 : iv_loop = loop->inner;
9631 : }
9632 : else
9633 : iv_loop = loop;
9634 168692 : gcc_assert (iv_loop == (gimple_bb (phi))->loop_father);
9635 :
9636 168692 : if (!nunits.is_constant () && SLP_TREE_LANES (slp_node) != 1)
9637 : {
9638 : /* The current SLP code creates the step value element-by-element. */
9639 : if (dump_enabled_p ())
9640 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9641 : "SLP induction not supported for variable-length"
9642 : " vectors.\n");
9643 : return false;
9644 : }
9645 :
9646 168692 : if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
9647 : {
9648 12 : if (dump_enabled_p ())
9649 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9650 : "floating point induction vectorization disabled\n");
9651 : return false;
9652 : }
9653 :
9654 168680 : tree step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info);
9655 168680 : gcc_assert (step_expr != NULL_TREE);
9656 337336 : if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr))
9657 337235 : && !type_has_mode_precision_p (TREE_TYPE (step_expr)))
9658 : {
9659 12 : if (dump_enabled_p ())
9660 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9661 : "bit-precision induction vectorization not "
9662 : "supported.\n");
9663 : return false;
9664 : }
9665 168668 : tree stept = TREE_TYPE (step_expr);
9666 168668 : tree step_vectype
9667 168668 : = (!INTEGRAL_TYPE_P (stept) ? vectype
9668 168567 : : signed_or_unsigned_type_for (TYPE_UNSIGNED (stept), vectype));
9669 168668 : stept = TREE_TYPE (step_vectype);
9670 :
9671 : /* Check for target support of the vectorized arithmetic used here. */
9672 168668 : if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
9673 168668 : || !target_supports_op_p (step_vectype, MINUS_EXPR, optab_default))
9674 : return false;
9675 138406 : if (!nunits.is_constant ()
9676 138406 : || !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9677 : {
9678 0 : if (!target_supports_op_p (step_vectype, MULT_EXPR, optab_default))
9679 : return false;
9680 : /* FLOAT_EXPR when computing VEC_INIT for float inductions. */
9681 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9682 : {
9683 0 : tree index_type = build_nonstandard_integer_type
9684 0 : (GET_MODE_BITSIZE (SCALAR_TYPE_MODE (stept)), 1);
9685 :
9686 0 : index_vectype = build_vector_type (index_type, nunits);
9687 0 : if (!can_float_p (TYPE_MODE (step_vectype),
9688 0 : TYPE_MODE (index_vectype), 1))
9689 : return false;
9690 : }
9691 : }
9692 :
9693 138406 : unsigned nvects = vect_get_num_copies (loop_vinfo, slp_node);
9694 138406 : if (cost_vec) /* transformation not required. */
9695 : {
9696 367488 : unsigned inside_cost = 0, prologue_cost = 0;
9697 : /* We eventually need to set a vector type on invariant
9698 : arguments. */
9699 : unsigned j;
9700 : slp_tree child;
9701 367488 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
9702 244992 : if (!vect_maybe_update_slp_op_vectype
9703 244992 : (child, SLP_TREE_VECTYPE (slp_node)))
9704 : {
9705 0 : if (dump_enabled_p ())
9706 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9707 : "incompatible vector types for "
9708 : "invariants\n");
9709 : return false;
9710 : }
9711 : /* loop cost for vec_loop. */
9712 122496 : inside_cost = record_stmt_cost (cost_vec, nvects,
9713 : vector_stmt, slp_node, 0, vect_body);
9714 : /* prologue cost for vec_init (if not nested) and step. */
9715 122496 : prologue_cost = record_stmt_cost (cost_vec, 1 + !nested_in_vect_loop,
9716 : scalar_to_vec,
9717 : slp_node, 0, vect_prologue);
9718 122496 : if (dump_enabled_p ())
9719 4285 : dump_printf_loc (MSG_NOTE, vect_location,
9720 : "vect_model_induction_cost: inside_cost = %d, "
9721 : "prologue_cost = %d .\n", inside_cost,
9722 : prologue_cost);
9723 :
9724 122496 : SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
9725 122496 : DUMP_VECT_SCOPE ("vectorizable_induction");
9726 122496 : return true;
9727 : }
9728 :
9729 : /* Transform. */
9730 :
9731 : /* Compute a vector variable, initialized with the first VF values of
9732 : the induction variable. E.g., for an iv with IV_PHI='X' and
9733 : evolution S, for a vector of 4 units, we want to compute:
9734 : [X, X + S, X + 2*S, X + 3*S]. */
9735 :
9736 15910 : if (dump_enabled_p ())
9737 2876 : dump_printf_loc (MSG_NOTE, vect_location, "transform induction phi.\n");
9738 :
9739 15910 : pe = loop_preheader_edge (iv_loop);
9740 : /* Find the first insertion point in the BB. */
9741 15910 : basic_block bb = gimple_bb (phi);
9742 15910 : si = gsi_after_labels (bb);
9743 :
9744 : /* For SLP induction we have to generate several IVs as for example
9745 : with group size 3 we need
9746 : [i0, i1, i2, i0 + S0] [i1 + S1, i2 + S2, i0 + 2*S0, i1 + 2*S1]
9747 : [i2 + 2*S2, i0 + 3*S0, i1 + 3*S1, i2 + 3*S2]. */
9748 15910 : gimple_stmt_iterator incr_si;
9749 15910 : bool insert_after;
9750 15910 : standard_iv_increment_position (iv_loop, &incr_si, &insert_after);
9751 :
9752 : /* The initial values are vectorized, but any lanes > group_size
9753 : need adjustment. */
9754 15910 : slp_tree init_node
9755 15910 : = SLP_TREE_CHILDREN (slp_node)[pe->dest_idx];
9756 :
9757 : /* Gather steps. Since we do not vectorize inductions as
9758 : cycles we have to reconstruct the step from SCEV data. */
9759 15910 : unsigned group_size = SLP_TREE_LANES (slp_node);
9760 15910 : tree *steps = XALLOCAVEC (tree, group_size);
9761 15910 : tree *inits = XALLOCAVEC (tree, group_size);
9762 15910 : stmt_vec_info phi_info;
9763 49156 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, phi_info)
9764 : {
9765 17336 : steps[i] = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
9766 17336 : if (!init_node)
9767 17103 : inits[i] = gimple_phi_arg_def (as_a<gphi *> (phi_info->stmt),
9768 : pe->dest_idx);
9769 : }
9770 :
9771 : /* Now generate the IVs. */
9772 31820 : gcc_assert (multiple_p (nunits * nvects, group_size));
9773 15910 : unsigned nivs;
9774 15910 : unsigned HOST_WIDE_INT const_nunits;
9775 15910 : if (nested_in_vect_loop)
9776 : nivs = nvects;
9777 15698 : else if (nunits.is_constant (&const_nunits)
9778 15698 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9779 : {
9780 15698 : gcc_assert (!init_node);
9781 : /* Compute the number of distinct IVs we need. We can reduce the
9782 : number when later vector chunks are equal to earlier chunks. */
9783 15698 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
9784 15698 : unsigned group_sizep = group_size;
9785 15698 : if (group_sizep % const_nunits == 0)
9786 : {
9787 122 : group_sizep = group_sizep / const_nunits;
9788 122 : unsigned candidate_nivs
9789 122 : = least_common_multiple (group_sizep, const_nunits) / const_nunits;
9790 122 : if (candidate_nivs < nivs
9791 122 : && vect_slp_induction_reuse_p (steps, inits, group_size,
9792 : const_nunits, candidate_nivs, nivs))
9793 : {
9794 20 : if (dump_enabled_p ())
9795 16 : dump_printf_loc (MSG_NOTE, vect_location,
9796 : "reusing %u SLP induction IVs for %u "
9797 : "vector chunks\n",
9798 : candidate_nivs, nivs);
9799 : nivs = candidate_nivs;
9800 : }
9801 : }
9802 : }
9803 : else
9804 : {
9805 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
9806 : nivs = 1;
9807 : }
9808 15910 : gimple_seq init_stmts = NULL;
9809 15910 : gimple_seq lupdate_mul_stmts = NULL;
9810 15910 : tree lupdate_mul = NULL_TREE;
9811 15910 : if (!nested_in_vect_loop)
9812 : {
9813 15698 : if (nunits.is_constant (&const_nunits)
9814 15698 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9815 : {
9816 : /* The number of iterations covered in one vector iteration. */
9817 15698 : unsigned lup_mul = (nvects * const_nunits) / group_size;
9818 15698 : lupdate_mul
9819 15698 : = build_vector_from_val (step_vectype,
9820 15698 : SCALAR_FLOAT_TYPE_P (stept)
9821 28 : ? build_real_from_wide (stept, lup_mul,
9822 : UNSIGNED)
9823 31368 : : build_int_cstu (stept, lup_mul));
9824 : }
9825 : else
9826 : {
9827 0 : gimple_seq *update_stmts
9828 : = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
9829 : ? &init_stmts
9830 : : &lupdate_mul_stmts;
9831 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9832 : {
9833 0 : tree increment
9834 0 : = gimple_convert (update_stmts, integer_type_node,
9835 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9836 0 : lupdate_mul = gimple_build (update_stmts, FLOAT_EXPR, stept,
9837 : increment);
9838 : }
9839 : else
9840 0 : lupdate_mul = gimple_convert (update_stmts, stept,
9841 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
9842 0 : lupdate_mul = gimple_build_vector_from_val (update_stmts,
9843 : step_vectype,
9844 : lupdate_mul);
9845 : }
9846 : }
9847 15910 : tree peel_mul = NULL_TREE;
9848 15910 : if (LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo))
9849 : {
9850 0 : if (SCALAR_FLOAT_TYPE_P (stept))
9851 0 : peel_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept,
9852 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9853 : else
9854 0 : peel_mul = gimple_convert (&init_stmts, stept,
9855 : LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo));
9856 0 : peel_mul = gimple_build_vector_from_val (&init_stmts,
9857 : step_vectype, peel_mul);
9858 : }
9859 15910 : tree step_mul = NULL_TREE;
9860 15910 : unsigned ivn;
9861 15910 : auto_vec<tree> vec_steps;
9862 32408 : for (ivn = 0; ivn < nivs; ++ivn)
9863 : {
9864 16498 : gimple_seq stmts = NULL;
9865 16498 : bool invariant = true;
9866 16498 : if (nunits.is_constant (&const_nunits)
9867 16498 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
9868 : {
9869 16498 : tree_vector_builder step_elts (step_vectype, const_nunits, 1);
9870 16498 : tree_vector_builder init_elts (vectype, const_nunits, 1);
9871 16498 : tree_vector_builder mul_elts (step_vectype, const_nunits, 1);
9872 121872 : for (unsigned eltn = 0; eltn < const_nunits; ++eltn)
9873 : {
9874 : /* The scalar steps of the IVs. */
9875 88876 : tree elt = steps[(ivn*const_nunits + eltn) % group_size];
9876 88876 : elt = gimple_convert (&init_stmts, TREE_TYPE (step_vectype), elt);
9877 88876 : step_elts.quick_push (elt);
9878 88876 : if (!init_node)
9879 : {
9880 : /* The scalar inits of the IVs if not vectorized. */
9881 87662 : elt = inits[(ivn*const_nunits + eltn) % group_size];
9882 87662 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
9883 87662 : TREE_TYPE (elt)))
9884 324 : elt = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9885 324 : TREE_TYPE (vectype), elt);
9886 87662 : init_elts.quick_push (elt);
9887 : }
9888 : /* The number of steps to add to the initial values. */
9889 88876 : unsigned mul_elt = (ivn*const_nunits + eltn) / group_size;
9890 177752 : mul_elts.quick_push (SCALAR_FLOAT_TYPE_P (stept)
9891 177650 : ? build_real_from_wide (stept, mul_elt,
9892 : UNSIGNED)
9893 177650 : : build_int_cstu (stept, mul_elt));
9894 : }
9895 16498 : vec_step = gimple_build_vector (&init_stmts, &step_elts);
9896 16498 : step_mul = gimple_build_vector (&init_stmts, &mul_elts);
9897 16498 : if (!init_node)
9898 16252 : vec_init = gimple_build_vector (&init_stmts, &init_elts);
9899 16498 : }
9900 : else
9901 : {
9902 0 : tree step = gimple_convert (&init_stmts, stept, steps[0]);
9903 0 : if (init_node)
9904 : ;
9905 0 : else if (INTEGRAL_TYPE_P (stept))
9906 : {
9907 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
9908 : /* Build the initial value directly as a VEC_SERIES_EXPR. */
9909 0 : vec_init = gimple_build (&init_stmts, VEC_SERIES_EXPR,
9910 : step_vectype, new_name, step);
9911 0 : if (!useless_type_conversion_p (vectype, step_vectype))
9912 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9913 : vectype, vec_init);
9914 : }
9915 : else
9916 : {
9917 : /* Build:
9918 : [base, base, base, ...]
9919 : + (vectype) [0, 1, 2, ...] * [step, step, step, ...]. */
9920 0 : gcc_assert (SCALAR_FLOAT_TYPE_P (stept));
9921 0 : gcc_assert (flag_associative_math);
9922 0 : gcc_assert (index_vectype != NULL_TREE);
9923 :
9924 0 : tree index = build_index_vector (index_vectype, 0, 1);
9925 0 : new_name = gimple_convert (&init_stmts, stept, inits[0]);
9926 0 : tree base_vec = gimple_build_vector_from_val (&init_stmts,
9927 : step_vectype,
9928 : new_name);
9929 0 : tree step_vec = gimple_build_vector_from_val (&init_stmts,
9930 : step_vectype,
9931 : step);
9932 0 : vec_init = gimple_build (&init_stmts, FLOAT_EXPR,
9933 : step_vectype, index);
9934 0 : vec_init = gimple_build (&init_stmts, MULT_EXPR,
9935 : step_vectype, vec_init, step_vec);
9936 0 : vec_init = gimple_build (&init_stmts, PLUS_EXPR,
9937 : step_vectype, vec_init, base_vec);
9938 0 : if (!useless_type_conversion_p (vectype, step_vectype))
9939 0 : vec_init = gimple_build (&init_stmts, VIEW_CONVERT_EXPR,
9940 : vectype, vec_init);
9941 : }
9942 : /* iv_loop is nested in the loop to be vectorized. Generate:
9943 : vec_step = [S, S, S, S] */
9944 0 : t = unshare_expr (step);
9945 0 : gcc_assert (CONSTANT_CLASS_P (t)
9946 : || TREE_CODE (t) == SSA_NAME);
9947 0 : vec_step = gimple_build_vector_from_val (&init_stmts,
9948 : step_vectype, t);
9949 : }
9950 16498 : vec_steps.safe_push (vec_step);
9951 16498 : if (peel_mul)
9952 : {
9953 0 : if (!step_mul)
9954 : {
9955 0 : gcc_assert (!nunits.is_constant ());
9956 : step_mul = gimple_build (&init_stmts,
9957 : MINUS_EXPR, step_vectype,
9958 : build_zero_cst (step_vectype), peel_mul);
9959 : }
9960 : else
9961 0 : step_mul = gimple_build (&init_stmts,
9962 : MINUS_EXPR, step_vectype,
9963 : step_mul, peel_mul);
9964 : }
9965 :
9966 : /* Create the induction-phi that defines the induction-operand. */
9967 16498 : vec_dest = vect_get_new_vect_var (vectype, vect_simple_var,
9968 : "vec_iv_");
9969 16498 : induction_phi = create_phi_node (vec_dest, iv_loop->header);
9970 16498 : induc_def = PHI_RESULT (induction_phi);
9971 :
9972 : /* Create the iv update inside the loop */
9973 16498 : tree up = vec_step;
9974 16498 : if (lupdate_mul)
9975 : {
9976 16252 : if (lupdate_mul_stmts)
9977 0 : gimple_seq_add_seq (&stmts, lupdate_mul_stmts);
9978 16252 : up = gimple_build (&stmts, MULT_EXPR, step_vectype, vec_step,
9979 : lupdate_mul);
9980 : }
9981 16498 : vec_def = gimple_convert (&stmts, step_vectype, induc_def);
9982 16498 : vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, up);
9983 16498 : vec_def = gimple_convert (&stmts, vectype, vec_def);
9984 16498 : insert_iv_increment (&incr_si, insert_after, stmts);
9985 16498 : add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
9986 : UNKNOWN_LOCATION);
9987 :
9988 16498 : if (init_node)
9989 246 : vec_init = vect_get_slp_vect_def (init_node, ivn);
9990 16498 : if (!nested_in_vect_loop
9991 16498 : && step_mul
9992 16498 : && !integer_zerop (step_mul))
9993 : {
9994 15784 : gcc_assert (invariant);
9995 15784 : vec_def = gimple_convert (&init_stmts, step_vectype, vec_init);
9996 15784 : up = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
9997 : vec_step, step_mul);
9998 15784 : vec_def = gimple_build (&init_stmts, PLUS_EXPR, step_vectype,
9999 : vec_def, up);
10000 15784 : vec_init = gimple_convert (&init_stmts, vectype, vec_def);
10001 : }
10002 :
10003 : /* Set the arguments of the phi node: */
10004 16498 : add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
10005 :
10006 16498 : slp_node->push_vec_def (induction_phi);
10007 : }
10008 15910 : if (!nested_in_vect_loop)
10009 : {
10010 : /* Fill up to the number of vectors we need for the whole group. */
10011 15698 : if (nunits.is_constant (&const_nunits)
10012 15698 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10013 15698 : nivs = least_common_multiple (group_size, const_nunits) / const_nunits;
10014 : else
10015 : nivs = 1;
10016 15698 : vec_steps.reserve (nivs-ivn);
10017 15698 : unsigned generated_nivs = ivn;
10018 15698 : gcc_assert (generated_nivs > 0);
10019 15734 : for (; ivn < nivs; ++ivn)
10020 : {
10021 36 : unsigned reuse_ivn = ivn % generated_nivs;
10022 36 : slp_node->push_vec_def (SLP_TREE_VEC_DEFS (slp_node)[reuse_ivn]);
10023 36 : vec_steps.quick_push (vec_steps[reuse_ivn]);
10024 : }
10025 : }
10026 :
10027 : /* Re-use IVs when we can. We are generating further vector
10028 : stmts by adding VF' * stride to the IVs generated above. */
10029 15910 : if (ivn < nvects)
10030 : {
10031 3516 : if (nunits.is_constant (&const_nunits)
10032 3516 : && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10033 : {
10034 3516 : unsigned vfp = (least_common_multiple (group_size, const_nunits)
10035 3516 : / group_size);
10036 3516 : lupdate_mul
10037 3516 : = build_vector_from_val (step_vectype,
10038 3516 : SCALAR_FLOAT_TYPE_P (stept)
10039 8 : ? build_real_from_wide (stept,
10040 8 : vfp, UNSIGNED)
10041 7024 : : build_int_cstu (stept, vfp));
10042 : }
10043 : else
10044 : {
10045 0 : if (SCALAR_FLOAT_TYPE_P (stept))
10046 : {
10047 0 : tree tem = build_int_cst (integer_type_node, nunits);
10048 0 : lupdate_mul = gimple_build (&init_stmts, FLOAT_EXPR, stept, tem);
10049 : }
10050 : else
10051 0 : lupdate_mul = build_int_cst (stept, nunits);
10052 0 : lupdate_mul = gimple_build_vector_from_val (&init_stmts, step_vectype,
10053 : lupdate_mul);
10054 : }
10055 11344 : for (; ivn < nvects; ++ivn)
10056 : {
10057 7828 : gimple *iv
10058 7828 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node)[ivn - nivs]);
10059 7828 : tree def = gimple_get_lhs (iv);
10060 7828 : if (ivn < 2*nivs)
10061 3614 : vec_steps[ivn - nivs]
10062 3614 : = gimple_build (&init_stmts, MULT_EXPR, step_vectype,
10063 3614 : vec_steps[ivn - nivs], lupdate_mul);
10064 7828 : gimple_seq stmts = NULL;
10065 7828 : def = gimple_convert (&stmts, step_vectype, def);
10066 23484 : def = gimple_build (&stmts, PLUS_EXPR, step_vectype,
10067 7828 : def, vec_steps[ivn % nivs]);
10068 7828 : def = gimple_convert (&stmts, vectype, def);
10069 7828 : if (gimple_code (iv) == GIMPLE_PHI)
10070 3614 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10071 : else
10072 : {
10073 4214 : gimple_stmt_iterator tgsi = gsi_for_stmt (iv);
10074 4214 : gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING);
10075 : }
10076 7828 : slp_node->push_vec_def (def);
10077 : }
10078 : }
10079 :
10080 15910 : new_bb = gsi_insert_seq_on_edge_immediate (pe, init_stmts);
10081 15910 : gcc_assert (!new_bb);
10082 :
10083 15910 : return true;
10084 15910 : }
10085 :
10086 : /* Function vectorizable_live_operation_1.
10087 :
10088 : helper function for vectorizable_live_operation. */
10089 :
10090 : static tree
10091 2772 : vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
10092 : tree vectype, slp_tree slp_node,
10093 : tree bitsize, tree bitstart, tree vec_lhs,
10094 : tree lhs_type, gimple_stmt_iterator *exit_gsi)
10095 : {
10096 2772 : gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10097 :
10098 2772 : tree vec_lhs_phi = copy_ssa_name (vec_lhs);
10099 2772 : gimple *phi = create_phi_node (vec_lhs_phi, exit_bb);
10100 8318 : for (unsigned i = 0; i < gimple_phi_num_args (phi); i++)
10101 2774 : SET_PHI_ARG_DEF (phi, i, vec_lhs);
10102 :
10103 2772 : gimple_seq stmts = NULL;
10104 2772 : tree new_tree;
10105 :
10106 : /* If bitstart is 0 then we can use a BIT_FIELD_REF */
10107 2772 : if (integer_zerop (bitstart))
10108 : {
10109 210 : tree scalar_res = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
10110 : vec_lhs_phi, bitsize, bitstart);
10111 :
10112 : /* Convert the extracted vector element to the scalar type. */
10113 210 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10114 : }
10115 2562 : else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10116 : {
10117 : /* Emit:
10118 :
10119 : SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>
10120 :
10121 : where VEC_LHS is the vectorized live-out result, LEN is the length of
10122 : the vector, BIAS is the load-store bias. The bias should not be used
10123 : at all since we are not using load/store operations, but LEN will be
10124 : REALLEN + BIAS, so subtract it to get to the correct position. */
10125 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10126 0 : gimple_seq tem = NULL;
10127 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10128 0 : tree len = vect_get_loop_len (loop_vinfo, &gsi,
10129 : &LOOP_VINFO_LENS (loop_vinfo),
10130 : 1, vectype, 0, 1, false);
10131 0 : gimple_seq_add_seq (&stmts, tem);
10132 :
10133 : /* LAST_INDEX = LEN - 1. */
10134 0 : tree last_index = gimple_build (&stmts, MINUS_EXPR, TREE_TYPE (len),
10135 0 : len, build_one_cst (TREE_TYPE (len)));
10136 :
10137 : /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN - 1>. */
10138 0 : tree scalar_res
10139 0 : = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
10140 : vec_lhs_phi, last_index);
10141 :
10142 : /* Convert the extracted vector element to the scalar type. */
10143 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10144 : }
10145 2562 : else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
10146 : {
10147 : /* Emit:
10148 :
10149 : SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
10150 :
10151 : where VEC_LHS is the vectorized live-out result and MASK is
10152 : the loop mask for the final iteration. */
10153 0 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
10154 0 : tree scalar_type = TREE_TYPE (vectype);
10155 0 : gimple_seq tem = NULL;
10156 0 : gimple_stmt_iterator gsi = gsi_last (tem);
10157 0 : tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
10158 : &LOOP_VINFO_MASKS (loop_vinfo),
10159 : 1, vectype, 0);
10160 0 : tree scalar_res;
10161 0 : gimple_seq_add_seq (&stmts, tem);
10162 :
10163 0 : scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type,
10164 : mask, vec_lhs_phi);
10165 :
10166 : /* Convert the extracted vector element to the scalar type. */
10167 0 : new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
10168 : }
10169 : else
10170 : {
10171 2562 : tree bftype = TREE_TYPE (vectype);
10172 2562 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10173 85 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10174 2562 : new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart);
10175 2562 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10176 : &stmts, true, NULL_TREE);
10177 : }
10178 :
10179 2772 : *exit_gsi = gsi_after_labels (exit_bb);
10180 2772 : if (stmts)
10181 2772 : gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT);
10182 :
10183 2772 : return new_tree;
10184 : }
10185 :
10186 : /* Function vectorizable_live_operation.
10187 :
10188 : STMT_INFO computes a value that is used outside the loop. Check if
10189 : it can be supported. */
10190 :
10191 : bool
10192 286729 : vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
10193 : slp_tree slp_node, slp_instance slp_node_instance,
10194 : int slp_index, bool vec_stmt_p,
10195 : stmt_vector_for_cost *cost_vec)
10196 : {
10197 286729 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
10198 286729 : imm_use_iterator imm_iter;
10199 286729 : tree lhs, lhs_type, bitsize;
10200 286729 : tree vectype = SLP_TREE_VECTYPE (slp_node);
10201 286729 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
10202 286729 : gimple *use_stmt;
10203 286729 : use_operand_p use_p;
10204 286729 : auto_vec<tree> vec_oprnds;
10205 286729 : int vec_entry = 0;
10206 286729 : poly_uint64 vec_index = 0;
10207 :
10208 286729 : gcc_assert (STMT_VINFO_LIVE_P (stmt_info)
10209 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo));
10210 :
10211 : /* If a stmt of a reduction is live, vectorize it via
10212 : vect_create_epilog_for_reduction. vectorizable_reduction assessed
10213 : validity so just trigger the transform here. */
10214 286729 : if (vect_is_reduction (slp_node))
10215 : {
10216 87667 : if (!vec_stmt_p)
10217 : {
10218 64223 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10219 64223 : return true;
10220 : }
10221 : /* For SLP reductions we vectorize the epilogue for all involved stmts
10222 : together. For SLP reduction chains we only get here once. */
10223 23444 : if (SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_group
10224 23144 : && slp_index != 0)
10225 : return true;
10226 22971 : vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, slp_node);
10227 22971 : if (VECT_REDUC_INFO_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
10228 22971 : || VECT_REDUC_INFO_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
10229 : return true;
10230 :
10231 22041 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10232 22041 : || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10233 22032 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info, slp_node,
10234 : slp_node_instance,
10235 : LOOP_VINFO_MAIN_EXIT (loop_vinfo));
10236 :
10237 : /* If early break we only have to materialize the reduction on the merge
10238 : block, but we have to find an alternate exit first. */
10239 22041 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
10240 : {
10241 28 : slp_tree phis_node = slp_node_instance->reduc_phis;
10242 28 : stmt_info = SLP_TREE_REPRESENTATIVE (phis_node);
10243 89 : for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
10244 28 : if (exit != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
10245 : {
10246 23 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10247 : phis_node, slp_node_instance,
10248 : exit);
10249 23 : break;
10250 28 : }
10251 28 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
10252 9 : vect_create_epilog_for_reduction (loop_vinfo, stmt_info,
10253 : phis_node, slp_node_instance,
10254 : LOOP_VINFO_MAIN_EXIT
10255 : (loop_vinfo));
10256 : }
10257 :
10258 : return true;
10259 : }
10260 :
10261 : /* If STMT is not relevant and it is a simple assignment and its inputs are
10262 : invariant then it can remain in place, unvectorized. The original last
10263 : scalar value that it computes will be used. */
10264 199062 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
10265 : {
10266 0 : gcc_assert (is_simple_and_all_uses_invariant (stmt_info, loop_vinfo));
10267 0 : if (dump_enabled_p ())
10268 0 : dump_printf_loc (MSG_NOTE, vect_location,
10269 : "statement is simple and uses invariant. Leaving in "
10270 : "place.\n");
10271 : return true;
10272 : }
10273 :
10274 199062 : gcc_assert (slp_index >= 0);
10275 :
10276 199062 : int num_vec = vect_get_num_copies (vinfo, slp_node);
10277 199062 : poly_uint64 pos = slp_index;
10278 199062 : if (loop_vinfo)
10279 : {
10280 : /* Get the last occurrence of the scalar index from the concatenation of
10281 : all the slp vectors. Calculate which slp vector it is and the index
10282 : within. */
10283 78634 : int num_scalar = SLP_TREE_LANES (slp_node);
10284 78634 : pos += (num_vec * nunits) - num_scalar;
10285 : }
10286 :
10287 : /* Calculate which vector contains the result, and which lane of
10288 : that vector we need. */
10289 199062 : if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
10290 : {
10291 : if (dump_enabled_p ())
10292 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10293 : "Cannot determine which vector holds the"
10294 : " final result.\n");
10295 : return false;
10296 : }
10297 :
10298 199062 : if (!vec_stmt_p)
10299 : {
10300 : /* No transformation required. */
10301 182059 : if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
10302 : {
10303 30491 : if (SLP_TREE_LANES (slp_node) != 1)
10304 : {
10305 19 : if (dump_enabled_p ())
10306 19 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10307 : "can't operate on partial vectors "
10308 : "because an SLP statement is live after "
10309 : "the loop.\n");
10310 19 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10311 : }
10312 30472 : else if (num_vec > 1)
10313 : {
10314 17355 : if (dump_enabled_p ())
10315 53 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10316 : "can't operate on partial vectors "
10317 : "because ncopies is greater than 1.\n");
10318 17355 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10319 : }
10320 : else
10321 : {
10322 13117 : if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
10323 : OPTIMIZE_FOR_SPEED))
10324 0 : vect_record_loop_mask (loop_vinfo,
10325 : &LOOP_VINFO_MASKS (loop_vinfo),
10326 : 1, vectype, NULL);
10327 13117 : else if (can_vec_extract_var_idx_p (
10328 13117 : TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
10329 0 : vect_record_loop_len (loop_vinfo,
10330 : &LOOP_VINFO_LENS (loop_vinfo),
10331 : 1, vectype, 1);
10332 : else
10333 : {
10334 13117 : if (dump_enabled_p ())
10335 695 : dump_printf_loc (
10336 695 : MSG_MISSED_OPTIMIZATION, vect_location,
10337 : "can't operate on partial vectors "
10338 : "because the target doesn't support extract "
10339 : "last reduction.\n");
10340 13117 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
10341 : }
10342 : }
10343 : }
10344 30491 : record_stmt_cost (cost_vec, 1, vec_to_scalar, slp_node,
10345 : 0, loop_vinfo ? vect_epilogue : vect_body);
10346 182059 : SLP_TREE_LIVE_LANES (slp_node).safe_push (slp_index);
10347 182059 : return true;
10348 : }
10349 :
10350 : /* Use the lhs of the original scalar statement. */
10351 17003 : gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
10352 17003 : if (dump_enabled_p ())
10353 1028 : dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
10354 : "stmt %G", stmt);
10355 :
10356 17003 : lhs = gimple_get_lhs (stmt);
10357 17003 : lhs_type = TREE_TYPE (lhs);
10358 :
10359 17003 : bitsize = vector_element_bits_tree (vectype);
10360 :
10361 : /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
10362 17003 : gcc_assert (!loop_vinfo
10363 : || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10364 : && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
10365 : || SLP_TREE_LANES (slp_node) == 1));
10366 :
10367 : /* Get the correct slp vectorized stmt. */
10368 17003 : tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
10369 :
10370 : /* In case we need to early break vectorize also get the first stmt. */
10371 17003 : tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
10372 :
10373 : /* Get entry to use. */
10374 17003 : tree bitstart = bitsize_int (vec_index);
10375 17003 : bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
10376 :
10377 17003 : if (loop_vinfo)
10378 : {
10379 : /* Ensure the VEC_LHS for lane extraction stmts satisfy loop-closed PHI
10380 : requirement, insert one phi node for it. It looks like:
10381 : loop;
10382 : BB:
10383 : # lhs' = PHI <lhs>
10384 : ==>
10385 : loop;
10386 : BB:
10387 : # vec_lhs' = PHI <vec_lhs>
10388 : new_tree = lane_extract <vec_lhs', ...>;
10389 : lhs' = new_tree; */
10390 :
10391 2843 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10392 : /* Check if we have a loop where the chosen exit is not the main exit,
10393 : in these cases for an early break we restart the iteration the vector code
10394 : did. For the live values we want the value at the start of the iteration
10395 : rather than at the end. */
10396 2843 : edge main_e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
10397 2843 : bool all_exits_as_early_p = LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo);
10398 11994 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10399 9151 : if (!is_gimple_debug (use_stmt)
10400 9151 : && !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
10401 2772 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10402 : {
10403 2772 : edge e = gimple_phi_arg_edge (as_a <gphi *> (use_stmt),
10404 2772 : phi_arg_index_from_use (use_p));
10405 2772 : gcc_assert (loop_exit_edge_p (loop, e));
10406 2772 : bool main_exit_edge = e == main_e;
10407 2772 : tree tmp_vec_lhs = vec_lhs;
10408 2772 : tree tmp_bitstart = bitstart;
10409 :
10410 : /* For early exit where the exit is not in the BB that leads
10411 : to the latch then we're restarting the iteration in the
10412 : scalar loop. So get the first live value. */
10413 2772 : bool early_break_first_element_p
10414 2772 : = all_exits_as_early_p || !main_exit_edge;
10415 2772 : if (early_break_first_element_p)
10416 : {
10417 192 : tmp_vec_lhs = vec_lhs0;
10418 192 : tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
10419 : }
10420 :
10421 2772 : gimple_stmt_iterator exit_gsi;
10422 2772 : tree new_tree
10423 2772 : = vectorizable_live_operation_1 (loop_vinfo,
10424 : e->dest, vectype,
10425 : slp_node, bitsize,
10426 : tmp_bitstart, tmp_vec_lhs,
10427 : lhs_type, &exit_gsi);
10428 :
10429 2772 : auto gsi = gsi_for_stmt (use_stmt);
10430 2772 : tree lhs_phi = gimple_phi_result (use_stmt);
10431 2772 : remove_phi_node (&gsi, false);
10432 2772 : gimple *copy = gimple_build_assign (lhs_phi, new_tree);
10433 2772 : gsi_insert_before (&exit_gsi, copy, GSI_SAME_STMT);
10434 2772 : break;
10435 2843 : }
10436 :
10437 : /* There a no further out-of-loop uses of lhs by LC-SSA construction. */
10438 9222 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10439 6379 : gcc_assert (is_gimple_debug (use_stmt)
10440 2843 : || flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)));
10441 : }
10442 : else
10443 : {
10444 : /* For basic-block vectorization simply insert the lane-extraction. */
10445 14160 : tree bftype = TREE_TYPE (vectype);
10446 14160 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
10447 18 : bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
10448 14160 : tree new_tree = build3 (BIT_FIELD_REF, bftype,
10449 : vec_lhs, bitsize, bitstart);
10450 14160 : gimple_seq stmts = NULL;
10451 14160 : new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
10452 : &stmts, true, NULL_TREE);
10453 14160 : if (TREE_CODE (new_tree) == SSA_NAME
10454 28320 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
10455 2 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = 1;
10456 14160 : gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
10457 14160 : if (TREE_CODE (vec_lhs) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (vec_lhs))
10458 0 : vinfo->insert_seq_on_entry (stmt_info, stmts);
10459 14160 : else if (is_a <gphi *> (vec_stmt))
10460 : {
10461 1488 : gimple_stmt_iterator si = gsi_after_labels (gimple_bb (vec_stmt));
10462 1488 : gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
10463 : }
10464 : else
10465 : {
10466 12672 : gimple_stmt_iterator si = gsi_for_stmt (vec_stmt);
10467 12672 : gsi_insert_seq_after (&si, stmts, GSI_SAME_STMT);
10468 : }
10469 :
10470 : /* Replace use of lhs with newly computed result. If the use stmt is a
10471 : single arg PHI, just replace all uses of PHI result. It's necessary
10472 : because lcssa PHI defining lhs may be before newly inserted stmt. */
10473 14160 : use_operand_p use_p;
10474 14160 : stmt_vec_info use_stmt_info;
10475 79031 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
10476 64871 : if (!is_gimple_debug (use_stmt)
10477 64871 : && (!(use_stmt_info = vinfo->lookup_stmt (use_stmt))
10478 48469 : || !PURE_SLP_STMT (use_stmt_info)))
10479 : {
10480 : /* ??? This can happen when the live lane ends up being
10481 : rooted in a vector construction code-generated by an
10482 : external SLP node (and code-generation for that already
10483 : happened).
10484 : Doing this is what would happen if that vector CTOR
10485 : were not code-generated yet so it is not too bad.
10486 : ??? In fact we'd likely want to avoid this situation
10487 : in the first place. */
10488 26731 : if (TREE_CODE (new_tree) == SSA_NAME
10489 26731 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10490 26731 : && gimple_code (use_stmt) != GIMPLE_PHI
10491 50068 : && !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (new_tree),
10492 : use_stmt))
10493 : {
10494 0 : if (dump_enabled_p ())
10495 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10496 : "Using original scalar computation for "
10497 : "live lane because use precedes vector "
10498 : "def\n");
10499 0 : continue;
10500 : }
10501 56156 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
10502 : {
10503 : /* ??? It can also happen that we end up pulling a def into
10504 : a loop where replacing out-of-loop uses would require
10505 : a new LC SSA PHI node. Retain the original scalar in
10506 : those cases as well. PR98064. */
10507 28078 : edge e;
10508 28078 : if (TREE_CODE (new_tree) == SSA_NAME
10509 28078 : && !SSA_NAME_IS_DEFAULT_DEF (new_tree)
10510 28078 : && TREE_CODE (vec_lhs) == SSA_NAME
10511 28078 : && !SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
10512 28078 : && (gimple_bb (use_stmt)->loop_father
10513 28078 : != gimple_bb (vec_stmt)->loop_father)
10514 : /* But a replacement in a LC PHI is OK. This happens
10515 : in gcc.dg/vect/bb-slp-57.c for example. */
10516 3155 : && (gimple_code (use_stmt) != GIMPLE_PHI
10517 1261 : || (((e = phi_arg_edge_from_use (use_p)), true)
10518 1261 : && !loop_exit_edge_p
10519 1261 : (gimple_bb (vec_stmt)->loop_father, e)))
10520 30741 : && !flow_loop_nested_p (gimple_bb (vec_stmt)->loop_father,
10521 2663 : gimple_bb (use_stmt)->loop_father))
10522 : {
10523 0 : if (dump_enabled_p ())
10524 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10525 : "Using original scalar computation for "
10526 : "live lane because there is an "
10527 : "out-of-loop definition for it\n");
10528 0 : continue;
10529 : }
10530 28078 : SET_USE (use_p, new_tree);
10531 : }
10532 26731 : update_stmt (use_stmt);
10533 14160 : }
10534 : }
10535 :
10536 : return true;
10537 286729 : }
10538 :
10539 : /* Given loop represented by LOOP_VINFO, return true if computation of
10540 : LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
10541 : otherwise. */
10542 :
10543 : static bool
10544 63468 : loop_niters_no_overflow (loop_vec_info loop_vinfo)
10545 : {
10546 63468 : gcc_assert (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo));
10547 :
10548 : /* Constant case. */
10549 63468 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
10550 : {
10551 37469 : tree cst_niters = LOOP_VINFO_NITERS (loop_vinfo);
10552 37469 : tree cst_nitersm1 = LOOP_VINFO_NITERSM1 (loop_vinfo);
10553 :
10554 37469 : gcc_assert (TREE_CODE (cst_niters) == INTEGER_CST);
10555 37469 : gcc_assert (TREE_CODE (cst_nitersm1) == INTEGER_CST);
10556 37469 : if (wi::to_widest (cst_nitersm1) < wi::to_widest (cst_niters))
10557 : return true;
10558 : }
10559 :
10560 25999 : widest_int max;
10561 25999 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10562 : /* Check the upper bound of loop niters. */
10563 25999 : if (get_max_loop_iterations (loop, &max))
10564 : {
10565 25999 : tree type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
10566 25999 : signop sgn = TYPE_SIGN (type);
10567 25999 : widest_int type_max = widest_int::from (wi::max_value (type), sgn);
10568 25999 : if (max < type_max)
10569 25824 : return true;
10570 25999 : }
10571 : return false;
10572 63468 : }
10573 :
10574 : /* Return a mask type with half the number of elements as OLD_TYPE,
10575 : given that it should have mode NEW_MODE. */
10576 :
10577 : tree
10578 4695 : vect_halve_mask_nunits (tree old_type, machine_mode new_mode)
10579 : {
10580 4695 : poly_uint64 nunits = exact_div (TYPE_VECTOR_SUBPARTS (old_type), 2);
10581 4695 : return build_truth_vector_type_for_mode (nunits, new_mode);
10582 : }
10583 :
10584 : /* Return a mask type with twice as many elements as OLD_TYPE,
10585 : given that it should have mode NEW_MODE. */
10586 :
10587 : tree
10588 7024 : vect_double_mask_nunits (tree old_type, machine_mode new_mode)
10589 : {
10590 7024 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (old_type) * 2;
10591 7024 : return build_truth_vector_type_for_mode (nunits, new_mode);
10592 : }
10593 :
10594 : /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
10595 : contain a sequence of NVECTORS masks that each control a vector of type
10596 : VECTYPE. If SCALAR_MASK is nonnull, the fully-masked loop would AND
10597 : these vector masks with the vector version of SCALAR_MASK. */
10598 :
10599 : void
10600 106446 : vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
10601 : unsigned int nvectors, tree vectype, tree scalar_mask)
10602 : {
10603 106446 : gcc_assert (nvectors != 0);
10604 :
10605 106446 : if (scalar_mask)
10606 : {
10607 4972 : scalar_cond_masked_key cond (scalar_mask, nvectors);
10608 4972 : loop_vinfo->scalar_cond_masked_set.add (cond);
10609 : }
10610 :
10611 106446 : masks->mask_set.add (std::make_pair (vectype, nvectors));
10612 106446 : }
10613 :
10614 : /* Given a complete set of masks MASKS, extract mask number INDEX
10615 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10616 : where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
10617 :
10618 : See the comment above vec_loop_masks for more details about the mask
10619 : arrangement. */
10620 :
10621 : tree
10622 214 : vect_get_loop_mask (loop_vec_info loop_vinfo,
10623 : gimple_stmt_iterator *gsi, vec_loop_masks *masks,
10624 : unsigned int nvectors, tree vectype, unsigned int index)
10625 : {
10626 214 : if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10627 : == vect_partial_vectors_while_ult)
10628 : {
10629 0 : rgroup_controls *rgm = &(masks->rgc_vec)[nvectors - 1];
10630 0 : tree mask_type = rgm->type;
10631 :
10632 : /* Populate the rgroup's mask array, if this is the first time we've
10633 : used it. */
10634 0 : if (rgm->controls.is_empty ())
10635 : {
10636 0 : rgm->controls.safe_grow_cleared (nvectors, true);
10637 0 : for (unsigned int i = 0; i < nvectors; ++i)
10638 : {
10639 0 : tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
10640 : /* Provide a dummy definition until the real one is available. */
10641 0 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10642 0 : rgm->controls[i] = mask;
10643 : }
10644 : }
10645 :
10646 0 : tree mask = rgm->controls[index];
10647 0 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
10648 0 : TYPE_VECTOR_SUBPARTS (vectype)))
10649 : {
10650 : /* A loop mask for data type X can be reused for data type Y
10651 : if X has N times more elements than Y and if Y's elements
10652 : are N times bigger than X's. In this case each sequence
10653 : of N elements in the loop mask will be all-zero or all-one.
10654 : We can then view-convert the mask so that each sequence of
10655 : N elements is replaced by a single element. */
10656 0 : gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type),
10657 : TYPE_VECTOR_SUBPARTS (vectype)));
10658 0 : gimple_seq seq = NULL;
10659 0 : mask_type = truth_type_for (vectype);
10660 0 : mask = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, mask);
10661 0 : if (seq)
10662 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10663 : }
10664 : return mask;
10665 : }
10666 214 : else if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
10667 : == vect_partial_vectors_avx512)
10668 : {
10669 : /* The number of scalars per iteration and the number of vectors are
10670 : both compile-time constants. */
10671 214 : unsigned int nscalars_per_iter
10672 214 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10673 214 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10674 :
10675 214 : rgroup_controls *rgm = &masks->rgc_vec[nscalars_per_iter - 1];
10676 :
10677 : /* The stored nV is dependent on the mask type produced. */
10678 214 : gcc_assert (exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10679 : TYPE_VECTOR_SUBPARTS (rgm->type)).to_constant ()
10680 : == rgm->factor);
10681 214 : nvectors = rgm->factor;
10682 :
10683 : /* Populate the rgroup's mask array, if this is the first time we've
10684 : used it. */
10685 214 : if (rgm->controls.is_empty ())
10686 : {
10687 23 : rgm->controls.safe_grow_cleared (nvectors, true);
10688 135 : for (unsigned int i = 0; i < nvectors; ++i)
10689 : {
10690 89 : tree mask = make_temp_ssa_name (rgm->type, NULL, "loop_mask");
10691 : /* Provide a dummy definition until the real one is available. */
10692 89 : SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
10693 89 : rgm->controls[i] = mask;
10694 : }
10695 : }
10696 214 : if (known_eq (TYPE_VECTOR_SUBPARTS (rgm->type),
10697 : TYPE_VECTOR_SUBPARTS (vectype)))
10698 166 : return rgm->controls[index];
10699 :
10700 : /* Split the vector if needed. Since we are dealing with integer mode
10701 : masks with AVX512 we can operate on the integer representation
10702 : performing the whole vector shifting. */
10703 48 : unsigned HOST_WIDE_INT factor;
10704 48 : bool ok = constant_multiple_p (TYPE_VECTOR_SUBPARTS (rgm->type),
10705 48 : TYPE_VECTOR_SUBPARTS (vectype), &factor);
10706 0 : gcc_assert (ok);
10707 48 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (rgm->type)) == MODE_INT);
10708 48 : tree mask_type = truth_type_for (vectype);
10709 48 : gcc_assert (GET_MODE_CLASS (TYPE_MODE (mask_type)) == MODE_INT);
10710 48 : unsigned vi = index / factor;
10711 48 : unsigned vpart = index % factor;
10712 48 : tree vec = rgm->controls[vi];
10713 48 : gimple_seq seq = NULL;
10714 48 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR,
10715 48 : lang_hooks.types.type_for_mode
10716 48 : (TYPE_MODE (rgm->type), 1), vec);
10717 : /* For integer mode masks simply shift the right bits into position. */
10718 48 : if (vpart != 0)
10719 40 : vec = gimple_build (&seq, RSHIFT_EXPR, TREE_TYPE (vec), vec,
10720 : build_int_cst (integer_type_node,
10721 80 : (TYPE_VECTOR_SUBPARTS (vectype)
10722 40 : * vpart)));
10723 48 : vec = gimple_convert (&seq, lang_hooks.types.type_for_mode
10724 48 : (TYPE_MODE (mask_type), 1), vec);
10725 48 : vec = gimple_build (&seq, VIEW_CONVERT_EXPR, mask_type, vec);
10726 48 : if (seq)
10727 48 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10728 : return vec;
10729 : }
10730 : else
10731 0 : gcc_unreachable ();
10732 : }
10733 :
10734 : /* Record that LOOP_VINFO would need LENS to contain a sequence of NVECTORS
10735 : lengths for controlling an operation on VECTYPE. The operation splits
10736 : each element of VECTYPE into FACTOR separate subelements, measuring the
10737 : length as a number of these subelements. */
10738 :
10739 : void
10740 0 : vect_record_loop_len (loop_vec_info loop_vinfo, vec_loop_lens *lens,
10741 : unsigned int nvectors, tree vectype, unsigned int factor)
10742 : {
10743 0 : gcc_assert (nvectors != 0);
10744 0 : if (lens->length () < nvectors)
10745 0 : lens->safe_grow_cleared (nvectors, true);
10746 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10747 :
10748 : /* The number of scalars per iteration, scalar occupied bytes and
10749 : the number of vectors are both compile-time constants. */
10750 0 : unsigned int nscalars_per_iter
10751 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10752 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10753 :
10754 0 : if (rgl->max_nscalars_per_iter < nscalars_per_iter)
10755 : {
10756 : /* For now, we only support cases in which all loads and stores fall back
10757 : to VnQI or none do. */
10758 0 : gcc_assert (!rgl->max_nscalars_per_iter
10759 : || (rgl->factor == 1 && factor == 1)
10760 : || (rgl->max_nscalars_per_iter * rgl->factor
10761 : == nscalars_per_iter * factor));
10762 0 : rgl->max_nscalars_per_iter = nscalars_per_iter;
10763 0 : rgl->type = vectype;
10764 0 : rgl->factor = factor;
10765 : }
10766 0 : }
10767 :
10768 : /* Given a complete set of lengths LENS, extract length number INDEX
10769 : for an rgroup that operates on NVECTORS vectors of type VECTYPE,
10770 : where 0 <= INDEX < NVECTORS. Return a value that contains FACTOR
10771 : multiplied by the number of elements that should be processed.
10772 : Insert any set-up statements before GSI. */
10773 :
10774 : tree
10775 0 : vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10776 : vec_loop_lens *lens, unsigned int nvectors, tree vectype,
10777 : unsigned int index, unsigned int factor, bool adjusted)
10778 : {
10779 0 : rgroup_controls *rgl = &(*lens)[nvectors - 1];
10780 0 : bool use_bias_adjusted_len =
10781 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo) != 0;
10782 :
10783 : /* Populate the rgroup's len array, if this is the first time we've
10784 : used it. */
10785 0 : if (rgl->controls.is_empty ())
10786 : {
10787 0 : rgl->controls.safe_grow_cleared (nvectors, true);
10788 0 : for (unsigned int i = 0; i < nvectors; ++i)
10789 : {
10790 0 : tree len_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
10791 0 : gcc_assert (len_type != NULL_TREE);
10792 :
10793 0 : tree len = make_temp_ssa_name (len_type, NULL, "loop_len");
10794 :
10795 : /* Provide a dummy definition until the real one is available. */
10796 0 : SSA_NAME_DEF_STMT (len) = gimple_build_nop ();
10797 0 : rgl->controls[i] = len;
10798 :
10799 0 : if (use_bias_adjusted_len)
10800 : {
10801 0 : gcc_assert (i == 0);
10802 0 : tree adjusted_len =
10803 0 : make_temp_ssa_name (len_type, NULL, "adjusted_loop_len");
10804 0 : SSA_NAME_DEF_STMT (adjusted_len) = gimple_build_nop ();
10805 0 : rgl->bias_adjusted_ctrl = adjusted_len;
10806 : }
10807 : }
10808 : }
10809 :
10810 0 : if (use_bias_adjusted_len && adjusted)
10811 0 : return rgl->bias_adjusted_ctrl;
10812 :
10813 0 : tree loop_len = rgl->controls[index];
10814 0 : if (rgl->factor == 1 && factor == 1)
10815 : {
10816 0 : poly_int64 nunits1 = TYPE_VECTOR_SUBPARTS (rgl->type);
10817 0 : poly_int64 nunits2 = TYPE_VECTOR_SUBPARTS (vectype);
10818 0 : if (maybe_ne (nunits1, nunits2))
10819 : {
10820 : /* A loop len for data type X can be reused for data type Y
10821 : if X has N times more elements than Y and if Y's elements
10822 : are N times bigger than X's. */
10823 0 : gcc_assert (multiple_p (nunits1, nunits2));
10824 0 : factor = exact_div (nunits1, nunits2).to_constant ();
10825 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10826 0 : gimple_seq seq = NULL;
10827 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10828 0 : build_int_cst (iv_type, factor));
10829 0 : if (seq)
10830 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10831 : }
10832 0 : }
10833 0 : else if (factor && rgl->factor != factor)
10834 : {
10835 : /* The number of scalars per iteration, scalar occupied bytes and
10836 : the number of vectors are both compile-time constants. */
10837 0 : unsigned int nscalars_per_iter
10838 0 : = exact_div (nvectors * TYPE_VECTOR_SUBPARTS (vectype),
10839 0 : LOOP_VINFO_VECT_FACTOR (loop_vinfo)).to_constant ();
10840 0 : unsigned int rglvecsize = rgl->factor * rgl->max_nscalars_per_iter;
10841 0 : unsigned int vecsize = nscalars_per_iter * factor;
10842 0 : if (rglvecsize > vecsize)
10843 : {
10844 0 : unsigned int fac = rglvecsize / vecsize;
10845 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10846 0 : gimple_seq seq = NULL;
10847 0 : loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
10848 0 : build_int_cst (iv_type, fac));
10849 0 : if (seq)
10850 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10851 : }
10852 0 : else if (rglvecsize < vecsize)
10853 : {
10854 0 : unsigned int fac = vecsize / rglvecsize;
10855 0 : tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
10856 0 : gimple_seq seq = NULL;
10857 0 : loop_len = gimple_build (&seq, MULT_EXPR, iv_type, loop_len,
10858 0 : build_int_cst (iv_type, fac));
10859 0 : if (seq)
10860 0 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
10861 : }
10862 : }
10863 : return loop_len;
10864 : }
10865 :
10866 : /* Generate the tree for the loop len mask and return it. Given the lens,
10867 : nvectors, vectype, index and factor to gen the len mask as below.
10868 :
10869 : tree len_mask = VCOND_MASK_LEN (compare_mask, ones, zero, len, bias)
10870 : */
10871 : tree
10872 0 : vect_gen_loop_len_mask (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
10873 : gimple_stmt_iterator *cond_gsi, vec_loop_lens *lens,
10874 : unsigned int nvectors, tree vectype, tree stmt,
10875 : unsigned int index, unsigned int factor)
10876 : {
10877 0 : tree all_one_mask = build_all_ones_cst (vectype);
10878 0 : tree all_zero_mask = build_zero_cst (vectype);
10879 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens, nvectors, vectype, index,
10880 : factor, true);
10881 0 : tree bias = build_int_cst (intQI_type_node,
10882 0 : LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo));
10883 0 : tree len_mask = make_temp_ssa_name (TREE_TYPE (stmt), NULL, "vec_len_mask");
10884 0 : gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5, stmt,
10885 : all_one_mask, all_zero_mask, len,
10886 : bias);
10887 0 : gimple_call_set_lhs (call, len_mask);
10888 0 : gsi_insert_before (cond_gsi, call, GSI_SAME_STMT);
10889 :
10890 0 : return len_mask;
10891 : }
10892 :
10893 : /* Update the exit profile counts scaling them from scalar to vector counts.
10894 :
10895 : The calculations are as follows.
10896 :
10897 : let P be the scalar probability that one iteration takes an early exit edge.
10898 : 1 - P is then the probabiliy that the exit is not taken.
10899 :
10900 : Since every vector iteration handles VF scalar iterations, the early exit
10901 : branch can be taken when any of the lanes are true.
10902 :
10903 : We invert this and first calculate the probablity that you reach the latch,
10904 : i.e. the probability that the vector code does not exit is the probability
10905 : Q that none of the lanes are true.
10906 :
10907 : Q = (1 - P) * (1 - P) * ... * (1 - P)
10908 : = (1 - P) ^ VF
10909 :
10910 : and so the probability that at least one lane breaks is 1 - Q,
10911 : or rather 1 - (i - P) ^ VF.
10912 :
10913 : For example, if the scalar break probability is 10% and VF is 4:
10914 :
10915 : P = 0.1
10916 : 1 - P = 0.9
10917 : (1 - P) ^ VF = 0.9 ^ 4 = 0.6561
10918 : 1 - (1 - p) ^ VF = 1 - 0.6561 = 0.3439
10919 :
10920 : So a scalar early-exit edge that is taken 10% of the time becomes a vector
10921 : early-exit edge that is taken about 34.4% of the time, because each vector
10922 : iteration gives the break condition four chances to trigger. */
10923 :
10924 : static void
10925 63511 : vect_update_early_break_profiles (loop_vec_info loop_vinfo)
10926 : {
10927 63511 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
10928 62074 : return;
10929 :
10930 1437 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
10931 1437 : unsigned int vf = vect_vf_for_cost (loop_vinfo);
10932 :
10933 1437 : auto_vec<std::pair<edge, profile_probability>, 8> updates;
10934 7301 : for (edge e : get_loop_exit_edges (loop))
10935 : {
10936 4360 : if ((e == LOOP_VINFO_MAIN_EXIT (loop_vinfo)
10937 1413 : && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
10938 3033 : || !e->probability.initialized_p ())
10939 1370 : continue;
10940 :
10941 : /* For any early break exits, the probablity of exiting are based on if
10942 : any lane is true, and so are tied to VF since you have VF chances.
10943 : As such scale the scalar profile by VF to the the probability for the
10944 : vector edges using 1 - (1 - p) ^ VF. */
10945 1620 : profile_probability probability
10946 1620 : = profile_probability::always () - e->probability.invert ().pow (vf);
10947 1620 : updates.safe_push ({e, probability});
10948 1437 : }
10949 :
10950 : /* First update the edge counts. */
10951 5931 : for (auto update : updates)
10952 1620 : set_edge_probability_and_rescale_others (update.first, update.second);
10953 :
10954 : /* And then only update the in loop BBs, i.e. the fall through block for
10955 : the early exits. */
10956 5931 : for (auto update : updates)
10957 8100 : for (edge e : update.first->src->succs)
10958 4860 : if (!loop_exit_edge_p (loop, e) && single_pred_p (e->dest))
10959 1620 : e->dest->count = e->count ();
10960 1437 : }
10961 :
10962 : /* Scale profiling counters by estimation for LOOP which is vectorized
10963 : by factor VF.
10964 : If FLAT is true, the loop we started with had unrealistically flat
10965 : profile. */
10966 :
10967 : static void
10968 63511 : scale_profile_for_vect_loop (loop_vec_info loop_vinfo,
10969 : class loop *loop, edge exit_e, unsigned vf,
10970 : bool flat)
10971 : {
10972 : /* First scale any early exits. */
10973 63511 : vect_update_early_break_profiles (loop_vinfo);
10974 :
10975 : /* For flat profiles do not scale down proportionally by VF and only
10976 : cap by known iteration count bounds. */
10977 63511 : if (flat)
10978 : {
10979 35672 : if (dump_file && (dump_flags & TDF_DETAILS))
10980 5513 : fprintf (dump_file,
10981 : "Vectorized loop profile seems flat; not scaling iteration "
10982 : "count down by the vectorization factor %i\n", vf);
10983 35672 : scale_loop_profile (loop, profile_probability::always (),
10984 : get_likely_max_loop_iterations_int (loop));
10985 35672 : return;
10986 : }
10987 : /* Loop body executes VF fewer times and exit increases VF times. */
10988 27839 : profile_count entry_count = loop_preheader_edge (loop)->count ();
10989 :
10990 : /* If we have unreliable loop profile avoid dropping entry
10991 : count below header count. This can happen since loops
10992 : has unrealistically low trip counts. */
10993 27839 : while (vf > 1
10994 28875 : && loop->header->count > entry_count
10995 58833 : && loop->header->count < entry_count * vf)
10996 : {
10997 2119 : if (dump_file && (dump_flags & TDF_DETAILS))
10998 158 : fprintf (dump_file,
10999 : "Vectorization factor %i seems too large for profile "
11000 : "previously believed to be consistent; reducing.\n", vf);
11001 2119 : vf /= 2;
11002 : }
11003 :
11004 27839 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
11005 : {
11006 27839 : if (entry_count.nonzero_p ())
11007 27839 : set_edge_probability_and_rescale_others
11008 27839 : (exit_e,
11009 27839 : entry_count.probability_in (loop->header->count / vf));
11010 : /* Avoid producing very large exit probability when we do not have
11011 : sensible profile. */
11012 0 : else if (exit_e->probability < profile_probability::always () / (vf * 2))
11013 0 : set_edge_probability_and_rescale_others (exit_e,
11014 : exit_e->probability * vf);
11015 : }
11016 :
11017 27839 : loop->latch->count = single_pred_edge (loop->latch)->count ();
11018 :
11019 27839 : scale_loop_profile (loop, profile_probability::always () / vf,
11020 : get_likely_max_loop_iterations_int (loop));
11021 : }
11022 :
11023 : /* Update EPILOGUE's loop_vec_info. EPILOGUE was constructed as a copy of the
11024 : original loop that has now been vectorized.
11025 :
11026 : The inits of the data_references need to be advanced with the number of
11027 : iterations of the main loop. This has been computed in vect_do_peeling and
11028 : is stored in parameter ADVANCE.
11029 :
11030 : Since the loop_vec_info of this EPILOGUE was constructed for the original
11031 : loop, its stmt_vec_infos all point to the original statements. These need
11032 : to be updated to point to their corresponding copies.
11033 :
11034 : The data_reference's connections also need to be updated. Their
11035 : corresponding dr_vec_info need to be reconnected to the EPILOGUE's
11036 : stmt_vec_infos, their statements need to point to their corresponding
11037 : copy. */
11038 :
11039 : static void
11040 6862 : update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
11041 : {
11042 6862 : loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
11043 6862 : hash_map<tree,tree> mapping;
11044 6862 : gimple *orig_stmt, *new_stmt;
11045 6862 : gimple_stmt_iterator epilogue_gsi;
11046 6862 : gphi_iterator epilogue_phi_gsi;
11047 6862 : stmt_vec_info stmt_vinfo = NULL, related_vinfo;
11048 6862 : basic_block *epilogue_bbs = get_loop_body (epilogue);
11049 6862 : unsigned i;
11050 :
11051 6862 : free (LOOP_VINFO_BBS (epilogue_vinfo));
11052 6862 : LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
11053 6862 : LOOP_VINFO_NBBS (epilogue_vinfo) = epilogue->num_nodes;
11054 :
11055 : /* The EPILOGUE loop is a copy of the original loop so they share the same
11056 : gimple UIDs. In this loop we update the loop_vec_info of the EPILOGUE to
11057 : point to the copied statements. */
11058 20586 : for (unsigned i = 0; i < epilogue->num_nodes; ++i)
11059 : {
11060 13724 : for (epilogue_phi_gsi = gsi_start_phis (epilogue_bbs[i]);
11061 35278 : !gsi_end_p (epilogue_phi_gsi); gsi_next (&epilogue_phi_gsi))
11062 : {
11063 21554 : new_stmt = epilogue_phi_gsi.phi ();
11064 :
11065 21554 : gcc_assert (gimple_uid (new_stmt) > 0);
11066 21554 : stmt_vinfo
11067 21554 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
11068 :
11069 21554 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
11070 : }
11071 :
11072 27448 : for (epilogue_gsi = gsi_start_bb (epilogue_bbs[i]);
11073 137372 : !gsi_end_p (epilogue_gsi); gsi_next (&epilogue_gsi))
11074 : {
11075 123648 : new_stmt = gsi_stmt (epilogue_gsi);
11076 123648 : if (is_gimple_debug (new_stmt))
11077 20053 : continue;
11078 :
11079 103595 : gcc_assert (gimple_uid (new_stmt) > 0);
11080 103595 : stmt_vinfo
11081 103595 : = epilogue_vinfo->stmt_vec_infos[gimple_uid (new_stmt) - 1];
11082 :
11083 103595 : STMT_VINFO_STMT (stmt_vinfo) = new_stmt;
11084 :
11085 103595 : related_vinfo = STMT_VINFO_RELATED_STMT (stmt_vinfo);
11086 103595 : if (related_vinfo != NULL && related_vinfo != stmt_vinfo)
11087 : {
11088 2218 : gimple *stmt = STMT_VINFO_STMT (related_vinfo);
11089 : /* Set BB such that the assert in
11090 : 'get_initial_defs_for_reduction' is able to determine that
11091 : the BB of the related stmt is inside this loop. */
11092 2218 : gimple_set_bb (stmt,
11093 : gimple_bb (new_stmt));
11094 2218 : related_vinfo = STMT_VINFO_RELATED_STMT (related_vinfo);
11095 2218 : gcc_assert (related_vinfo == NULL
11096 : || related_vinfo == stmt_vinfo);
11097 : }
11098 : }
11099 : }
11100 :
11101 6862 : struct data_reference *dr;
11102 6862 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (epilogue_vinfo);
11103 31170 : FOR_EACH_VEC_ELT (datarefs, i, dr)
11104 : {
11105 24308 : orig_stmt = DR_STMT (dr);
11106 24308 : gcc_assert (gimple_uid (orig_stmt) > 0);
11107 24308 : stmt_vinfo = epilogue_vinfo->stmt_vec_infos[gimple_uid (orig_stmt) - 1];
11108 24308 : DR_STMT (dr) = STMT_VINFO_STMT (stmt_vinfo);
11109 : }
11110 :
11111 : /* Advance data_reference's with the number of iterations of the previous
11112 : loop and its prologue. */
11113 6862 : vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
11114 :
11115 : /* Remember the advancement made. */
11116 6862 : LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
11117 6862 : }
11118 :
11119 : /* When vectorizing early break statements instructions that happen before
11120 : the early break in the current BB need to be moved to after the early
11121 : break. This function deals with that and assumes that any validity
11122 : checks has already been performed.
11123 :
11124 : While moving the instructions if it encounters a VUSE or VDEF it then
11125 : corrects the VUSES as it moves the statements along. GDEST is the location
11126 : in which to insert the new statements. */
11127 :
11128 : static void
11129 1437 : move_early_exit_stmts (loop_vec_info loop_vinfo)
11130 : {
11131 1437 : DUMP_VECT_SCOPE ("move_early_exit_stmts");
11132 :
11133 1437 : if (LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).is_empty ())
11134 1199 : return;
11135 :
11136 : /* Move all stmts that need moving. */
11137 238 : basic_block dest_bb = LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo);
11138 238 : gimple_stmt_iterator dest_gsi = gsi_after_labels (dest_bb);
11139 :
11140 238 : tree last_seen_vuse = NULL_TREE;
11141 575 : for (gimple *stmt : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo))
11142 : {
11143 : /* We have to update crossed degenerate virtual PHIs. Simply
11144 : elide them. */
11145 337 : if (gphi *vphi = dyn_cast <gphi *> (stmt))
11146 : {
11147 7 : tree vdef = gimple_phi_result (vphi);
11148 7 : tree vuse = gimple_phi_arg_def (vphi, 0);
11149 7 : imm_use_iterator iter;
11150 7 : use_operand_p use_p;
11151 7 : gimple *use_stmt;
11152 23 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
11153 : {
11154 32 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11155 16 : SET_USE (use_p, vuse);
11156 7 : }
11157 7 : auto gsi = gsi_for_stmt (stmt);
11158 7 : remove_phi_node (&gsi, true);
11159 7 : last_seen_vuse = vuse;
11160 7 : continue;
11161 7 : }
11162 :
11163 : /* Check to see if statement is still required for vect or has been
11164 : elided. */
11165 330 : auto stmt_info = loop_vinfo->lookup_stmt (stmt);
11166 330 : if (!stmt_info)
11167 0 : continue;
11168 :
11169 330 : if (dump_enabled_p ())
11170 165 : dump_printf_loc (MSG_NOTE, vect_location, "moving stmt %G", stmt);
11171 :
11172 330 : gimple_stmt_iterator stmt_gsi = gsi_for_stmt (stmt);
11173 330 : gsi_move_before (&stmt_gsi, &dest_gsi, GSI_NEW_STMT);
11174 660 : last_seen_vuse = gimple_vuse (stmt);
11175 : }
11176 :
11177 : /* Update all the stmts with their new reaching VUSES. */
11178 793 : for (auto p : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
11179 : {
11180 245 : if (dump_enabled_p ())
11181 167 : dump_printf_loc (MSG_NOTE, vect_location,
11182 : "updating vuse to %T for load %G",
11183 : last_seen_vuse, p);
11184 245 : gimple_set_vuse (p, last_seen_vuse);
11185 245 : update_stmt (p);
11186 : }
11187 :
11188 : /* And update the LC PHIs on exits. */
11189 1203 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
11190 489 : if (!dominated_by_p (CDI_DOMINATORS, e->src, dest_bb))
11191 269 : if (gphi *phi = get_virtual_phi (e->dest))
11192 507 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, last_seen_vuse);
11193 : }
11194 :
11195 : /* Generate adjustment code for early break scalar IVs filling in the value
11196 : we created earlier on for LOOP_VINFO_EARLY_BRK_NITERS_VAR. */
11197 :
11198 : static void
11199 1437 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
11200 : {
11201 1437 : DUMP_VECT_SCOPE ("vect_update_ivs_after_vectorizer_for_early_breaks");
11202 :
11203 1437 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
11204 : /* If no peeling was done then we have no IV to update. */
11205 1437 : || !LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo))
11206 596 : return;
11207 :
11208 841 : tree phi_var = LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo);
11209 841 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
11210 841 : tree ty_var = TREE_TYPE (phi_var);
11211 841 : auto loop = LOOP_VINFO_LOOP (loop_vinfo);
11212 841 : tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
11213 :
11214 : /* Remove the existing dummy GIMPLE statement and just keep the def. */
11215 841 : gimple *def = SSA_NAME_DEF_STMT (phi_var);
11216 841 : auto def_gsi = gsi_for_stmt (def);
11217 841 : gsi_remove (&def_gsi, true);
11218 :
11219 841 : auto induction_phi = create_phi_node (induc_var, loop->header);
11220 841 : tree induc_def = PHI_RESULT (induction_phi);
11221 :
11222 : /* Create the iv update inside the loop. */
11223 841 : gimple_seq init_stmts = NULL;
11224 841 : gimple_seq stmts = NULL;
11225 841 : gimple_seq iv_stmts = NULL;
11226 841 : tree tree_iv_incr = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
11227 :
11228 841 : tree iter_var;
11229 841 : if (POINTER_TYPE_P (ty_var))
11230 0 : iter_var = gimple_build (&stmts, POINTER_PLUS_EXPR, ty_var, induc_def,
11231 : tree_iv_incr);
11232 : else
11233 : {
11234 841 : tree offset = gimple_convert (&stmts, ty_var, tree_iv_incr);
11235 841 : iter_var = gimple_build (&stmts, PLUS_EXPR, ty_var, induc_def, offset);
11236 : }
11237 :
11238 841 : tree init_var = build_zero_cst (ty_var);
11239 841 : if (niters_skip)
11240 0 : init_var = gimple_build (&init_stmts, MINUS_EXPR, ty_var, init_var,
11241 : gimple_convert (&init_stmts, ty_var, niters_skip));
11242 :
11243 841 : add_phi_arg (induction_phi, iter_var,
11244 : loop_latch_edge (loop), UNKNOWN_LOCATION);
11245 841 : add_phi_arg (induction_phi, init_var,
11246 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
11247 :
11248 : /* Find the first insertion point in the BB. */
11249 841 : auto pe = loop_preheader_edge (loop);
11250 :
11251 : /* If we've done any peeling, calculate the peeling adjustment needed to the
11252 : final IV. */
11253 841 : if (niters_skip)
11254 : {
11255 0 : tree induc_type = TREE_TYPE (induc_def);
11256 0 : tree s_induc_type = signed_type_for (induc_type);
11257 0 : induc_def = gimple_build (&iv_stmts, MAX_EXPR, s_induc_type,
11258 : gimple_convert (&iv_stmts, s_induc_type,
11259 : induc_def),
11260 : build_zero_cst (s_induc_type));
11261 0 : auto stmt = gimple_build_assign (phi_var,
11262 : gimple_convert (&iv_stmts, induc_type,
11263 : induc_def));
11264 0 : gimple_seq_add_stmt_without_update (&iv_stmts, stmt);
11265 0 : basic_block exit_bb = NULL;
11266 : /* Identify the early exit merge block. I wish we had stored this. */
11267 0 : for (auto e : get_loop_exit_edges (loop))
11268 0 : if (e != LOOP_VINFO_MAIN_EXIT (loop_vinfo))
11269 : {
11270 0 : exit_bb = e->dest;
11271 0 : break;
11272 0 : }
11273 :
11274 0 : gcc_assert (exit_bb);
11275 0 : auto exit_gsi = gsi_after_labels (exit_bb);
11276 0 : gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
11277 : }
11278 : /* Write the init_stmts in the loop-preheader block. */
11279 841 : auto psi = gsi_last_nondebug_bb (pe->src);
11280 841 : gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);
11281 :
11282 : /* Write the adjustments at the end of the iv increment. */
11283 841 : bool insert_after;
11284 841 : gimple_stmt_iterator incr_gsi;
11285 841 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
11286 :
11287 841 : if (insert_after)
11288 0 : gsi_insert_seq_after (&incr_gsi, stmts, GSI_NEW_STMT);
11289 : else
11290 841 : gsi_insert_seq_before (&incr_gsi, stmts, GSI_NEW_STMT);
11291 : }
11292 :
11293 : /* Function vect_transform_loop.
11294 :
11295 : The analysis phase has determined that the loop is vectorizable.
11296 : Vectorize the loop - created vectorized stmts to replace the scalar
11297 : stmts in the loop, and update the loop exit condition.
11298 : Returns scalar epilogue loop if any. */
11299 :
11300 : class loop *
11301 63511 : vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
11302 : {
11303 63511 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11304 63511 : class loop *epilogue = NULL;
11305 63511 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
11306 63511 : int nbbs = loop->num_nodes;
11307 63511 : int i;
11308 63511 : tree niters_vector = NULL_TREE;
11309 63511 : tree step_vector = NULL_TREE;
11310 63511 : tree niters_vector_mult_vf = NULL_TREE;
11311 63511 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11312 63511 : unsigned int lowest_vf = constant_lower_bound (vf);
11313 63511 : gimple *stmt;
11314 63511 : bool check_profitability = false;
11315 63511 : unsigned int th;
11316 63511 : bool flat = maybe_flat_loop_profile (loop);
11317 63511 : bool uncounted_p = LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo);
11318 :
11319 63511 : DUMP_VECT_SCOPE ("vec_transform_loop");
11320 :
11321 63511 : if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11322 56649 : loop_vinfo->shared->check_datarefs ();
11323 :
11324 : /* Use the more conservative vectorization threshold. If the number
11325 : of iterations is constant assume the cost check has been performed
11326 : by our caller. If the threshold makes all loops profitable that
11327 : run at least the (estimated) vectorization factor number of times
11328 : checking is pointless, too. */
11329 63511 : th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
11330 63511 : if (vect_apply_runtime_profitability_check_p (loop_vinfo))
11331 : {
11332 18816 : if (dump_enabled_p ())
11333 180 : dump_printf_loc (MSG_NOTE, vect_location,
11334 : "Profitability threshold is %d loop iterations.\n",
11335 : th);
11336 : check_profitability = true;
11337 : }
11338 :
11339 : /* Make sure there exists a single-predecessor exit bb. Do this before
11340 : versioning. */
11341 63511 : edge e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
11342 63511 : if (! single_pred_p (e->dest) && !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11343 : {
11344 18979 : split_loop_exit_edge (e, true);
11345 18979 : if (dump_enabled_p ())
11346 2386 : dump_printf (MSG_NOTE, "split exit edge\n");
11347 : }
11348 :
11349 : /* Version the loop first, if required, so the profitability check
11350 : comes first. */
11351 :
11352 63511 : if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
11353 : {
11354 3809 : class loop *sloop
11355 3809 : = vect_loop_versioning (loop_vinfo, loop_vectorized_call);
11356 3809 : sloop->force_vectorize = false;
11357 3809 : check_profitability = false;
11358 : }
11359 :
11360 : /* Make sure there exists a single-predecessor exit bb also on the
11361 : scalar loop copy. Do this after versioning but before peeling
11362 : so CFG structure is fine for both scalar and if-converted loop
11363 : to make slpeel_duplicate_current_defs_from_edges face matched
11364 : loop closed PHI nodes on the exit. */
11365 63511 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
11366 : {
11367 7269 : e = LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo);
11368 7269 : if (! single_pred_p (e->dest))
11369 : {
11370 6979 : split_loop_exit_edge (e, true);
11371 6979 : if (dump_enabled_p ())
11372 1199 : dump_printf (MSG_NOTE, "split exit edge of scalar loop\n");
11373 : }
11374 : }
11375 :
11376 63511 : tree niters = vect_build_loop_niters (loop_vinfo);
11377 63511 : LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = niters;
11378 63511 : tree nitersm1 = unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo));
11379 63511 : tree advance;
11380 63511 : drs_init_vec orig_drs_init;
11381 63511 : bool niters_no_overflow = uncounted_p ? false /* Not known. */
11382 63468 : : loop_niters_no_overflow (loop_vinfo);
11383 :
11384 63511 : epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
11385 : &step_vector, &niters_vector_mult_vf, th,
11386 : check_profitability, niters_no_overflow,
11387 : &advance);
11388 :
11389 63511 : LOOP_VINFO_IV_INCREMENT (loop_vinfo)
11390 63511 : = vect_get_loop_iv_increment (loop_vinfo);
11391 :
11392 : /* Assign hierarchical discriminators to the vectorized loop. */
11393 63511 : poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11394 63511 : unsigned int vf_int = constant_lower_bound (vf_val);
11395 63511 : if (vf_int > DISCR_MULTIPLICITY_MAX)
11396 : vf_int = DISCR_MULTIPLICITY_MAX;
11397 :
11398 : /* Assign unique copy_id dynamically instead of using hardcoded constants.
11399 : Epilogue and main vectorized loops get different copy_ids. */
11400 63511 : gimple *loop_last = last_nondebug_stmt (loop->header);
11401 63511 : location_t loop_loc
11402 63511 : = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION;
11403 63231 : if (loop_loc != UNKNOWN_LOCATION)
11404 : {
11405 52597 : unsigned int copyid = allocate_copyid_base (loop_loc, 1);
11406 52597 : assign_discriminators_to_loop (loop, vf_int, copyid);
11407 : }
11408 63511 : if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
11409 63511 : && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
11410 : {
11411 : /* Ifcvt duplicates loop preheader, loop body and produces an basic
11412 : block after loop exit. We need to scale all that. */
11413 90 : basic_block preheader
11414 90 : = loop_preheader_edge (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))->src;
11415 90 : preheader->count
11416 : = preheader->count.apply_probability
11417 90 : (LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11418 90 : scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo),
11419 : LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo));
11420 90 : LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo)->dest->count = preheader->count;
11421 : }
11422 :
11423 63511 : if (niters_vector == NULL_TREE && !uncounted_p)
11424 : {
11425 28907 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
11426 28907 : && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11427 58618 : && known_eq (lowest_vf, vf))
11428 : {
11429 28902 : niters_vector
11430 28902 : = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
11431 28902 : LOOP_VINFO_INT_NITERS (loop_vinfo) / lowest_vf);
11432 28902 : step_vector = build_one_cst (TREE_TYPE (niters));
11433 : }
11434 814 : else if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11435 2 : vect_gen_vector_loop_niters (loop_vinfo, niters, &niters_vector,
11436 : &step_vector, niters_no_overflow);
11437 : else
11438 : /* vect_do_peeling subtracted the number of peeled prologue
11439 : iterations from LOOP_VINFO_NITERS. */
11440 812 : vect_gen_vector_loop_niters (loop_vinfo, LOOP_VINFO_NITERS (loop_vinfo),
11441 : &niters_vector, &step_vector,
11442 : niters_no_overflow);
11443 : }
11444 :
11445 : /* 1) Make sure the loop header has exactly two entries
11446 : 2) Make sure we have a preheader basic block. */
11447 :
11448 63511 : gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
11449 :
11450 63511 : split_edge (loop_preheader_edge (loop));
11451 :
11452 63511 : if (vect_use_loop_mask_for_alignment_p (loop_vinfo))
11453 : /* This will deal with any possible peeling. */
11454 2 : vect_prepare_for_masked_peels (loop_vinfo);
11455 :
11456 : /* Handle any code motion that we need to for early-break vectorization after
11457 : we've done peeling but just before we start vectorizing. */
11458 63511 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11459 : {
11460 1437 : vect_update_ivs_after_vectorizer_for_early_breaks (loop_vinfo);
11461 1437 : move_early_exit_stmts (loop_vinfo);
11462 : }
11463 :
11464 : /* Remove existing clobber stmts and prefetches. */
11465 193844 : for (i = 0; i < nbbs; i++)
11466 : {
11467 130333 : basic_block bb = bbs[i];
11468 1129374 : for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);)
11469 : {
11470 868708 : stmt = gsi_stmt (si);
11471 868708 : if (gimple_clobber_p (stmt)
11472 868708 : || gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH))
11473 : {
11474 102 : unlink_stmt_vdef (stmt);
11475 102 : gsi_remove (&si, true);
11476 102 : release_defs (stmt);
11477 : }
11478 : else
11479 868606 : gsi_next (&si);
11480 : }
11481 : }
11482 :
11483 : /* Schedule the SLP instances. */
11484 63511 : if (!loop_vinfo->slp_instances.is_empty ())
11485 : {
11486 63511 : DUMP_VECT_SCOPE ("scheduling SLP instances");
11487 63511 : vect_schedule_slp (loop_vinfo, LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
11488 : false);
11489 : }
11490 :
11491 : /* Generate the loop invariant statements. */
11492 63511 : if (!gimple_seq_empty_p (LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo)))
11493 : {
11494 80 : if (dump_enabled_p ())
11495 26 : dump_printf_loc (MSG_NOTE, vect_location,
11496 : "------>generating loop invariant statements\n");
11497 80 : gimple_stmt_iterator gsi;
11498 80 : gsi = gsi_after_labels (loop_preheader_edge (loop)->src);
11499 80 : gsi_insert_seq_before (&gsi, LOOP_VINFO_INV_PATTERN_DEF_SEQ (loop_vinfo),
11500 : GSI_CONTINUE_LINKING);
11501 : }
11502 :
11503 : /* Stub out scalar statements that must not survive vectorization and
11504 : were not picked as relevant in any SLP instance.
11505 : Doing this here helps with grouped statements, or statements that
11506 : are involved in patterns. */
11507 193844 : for (i = 0; i < nbbs; i++)
11508 : {
11509 130333 : basic_block bb = bbs[i];
11510 130333 : stmt_vec_info stmt_info;
11511 260666 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
11512 1729026 : !gsi_end_p (gsi); gsi_next (&gsi))
11513 : {
11514 1598693 : gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
11515 6370 : if (!call || !gimple_call_internal_p (call))
11516 1593475 : continue;
11517 5218 : internal_fn ifn = gimple_call_internal_fn (call);
11518 5218 : if (ifn == IFN_MASK_LOAD)
11519 : {
11520 682 : tree lhs = gimple_get_lhs (call);
11521 682 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11522 : {
11523 0 : tree zero = build_zero_cst (TREE_TYPE (lhs));
11524 0 : gimple *new_stmt = gimple_build_assign (lhs, zero);
11525 0 : gsi_replace (&gsi, new_stmt, true);
11526 : }
11527 : }
11528 4536 : else if (conditional_internal_fn_code (ifn) != ERROR_MARK)
11529 : {
11530 2291 : tree lhs = gimple_get_lhs (call);
11531 2291 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11532 : {
11533 0 : tree else_arg
11534 0 : = gimple_call_arg (call, gimple_call_num_args (call) - 1);
11535 0 : gimple *new_stmt = gimple_build_assign (lhs, else_arg);
11536 0 : gsi_replace (&gsi, new_stmt, true);
11537 : }
11538 : }
11539 2245 : else if (ifn == IFN_MASK_CALL
11540 4 : && (stmt_info = loop_vinfo->lookup_stmt (call))
11541 4 : && !STMT_VINFO_RELEVANT_P (stmt_info)
11542 2249 : && !STMT_VINFO_LIVE_P (stmt_info))
11543 : {
11544 4 : gcc_assert (!gimple_call_lhs (stmt_info->stmt));
11545 4 : loop_vinfo->remove_stmt (stmt_info);
11546 : }
11547 : }
11548 : }
11549 :
11550 63511 : if (!uncounted_p)
11551 : {
11552 : /* The vectorization factor is always > 1, so if we use an IV increment of
11553 : 1. A zero NITERS becomes a nonzero NITERS_VECTOR. */
11554 63468 : if (integer_onep (step_vector))
11555 63447 : niters_no_overflow = true;
11556 :
11557 63468 : vect_set_loop_condition (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
11558 : loop_vinfo, niters_vector, step_vector,
11559 : niters_vector_mult_vf, !niters_no_overflow);
11560 : }
11561 :
11562 63511 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
11563 :
11564 : /* True if the final iteration might not handle a full vector's
11565 : worth of scalar iterations. */
11566 127022 : bool final_iter_may_be_partial
11567 63511 : = LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
11568 63511 : || LOOP_VINFO_EARLY_BREAKS (loop_vinfo);
11569 :
11570 : /* +1 to convert latch counts to loop iteration counts. */
11571 63511 : int bias_for_lowest = 1;
11572 :
11573 : /* When we are peeling for gaps then we take away one scalar iteration
11574 : from the vector loop. Thus we can adjust the upper bound by one
11575 : scalar iteration. But only when we know the bound applies to the
11576 : IV exit test which might not be true when we have multiple exits. */
11577 63511 : if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
11578 123742 : bias_for_lowest -= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 0;
11579 :
11580 63511 : int bias_for_assumed = bias_for_lowest;
11581 63511 : int alignment_npeels = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
11582 63511 : if (alignment_npeels && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
11583 : {
11584 : /* When the amount of peeling is known at compile time, the first
11585 : iteration will have exactly alignment_npeels active elements.
11586 : In the worst case it will have at least one. */
11587 2 : int min_first_active = (alignment_npeels > 0 ? alignment_npeels : 1);
11588 2 : bias_for_lowest += lowest_vf - min_first_active;
11589 2 : bias_for_assumed += assumed_vf - min_first_active;
11590 : }
11591 : /* In these calculations the "- 1" converts loop iteration counts
11592 : back to latch counts. */
11593 63511 : if (loop->any_upper_bound)
11594 : {
11595 63495 : loop_vec_info main_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
11596 63495 : loop->nb_iterations_upper_bound
11597 63495 : = (final_iter_may_be_partial
11598 64935 : ? wi::udiv_ceil (loop->nb_iterations_upper_bound + bias_for_lowest,
11599 2880 : lowest_vf) - 1
11600 62055 : : wi::udiv_floor (loop->nb_iterations_upper_bound + bias_for_lowest,
11601 124110 : lowest_vf) - 1);
11602 63495 : if (main_vinfo
11603 : /* Both peeling for alignment and peeling for gaps can end up
11604 : with the scalar epilogue running for more than VF-1 iterations. */
11605 6862 : && !main_vinfo->peeling_for_alignment
11606 6814 : && !main_vinfo->peeling_for_gaps)
11607 : {
11608 6632 : unsigned int bound;
11609 6632 : poly_uint64 main_iters
11610 6632 : = upper_bound (LOOP_VINFO_VECT_FACTOR (main_vinfo),
11611 : LOOP_VINFO_COST_MODEL_THRESHOLD (main_vinfo));
11612 6632 : main_iters
11613 6632 : = upper_bound (main_iters,
11614 6632 : LOOP_VINFO_VERSIONING_THRESHOLD (main_vinfo));
11615 6632 : if (can_div_away_from_zero_p (main_iters,
11616 6632 : LOOP_VINFO_VECT_FACTOR (loop_vinfo),
11617 : &bound))
11618 6632 : loop->nb_iterations_upper_bound
11619 6632 : = wi::umin ((bound_wide_int) (bound - 1),
11620 6632 : loop->nb_iterations_upper_bound);
11621 : }
11622 : }
11623 63511 : if (loop->any_likely_upper_bound)
11624 63495 : loop->nb_iterations_likely_upper_bound
11625 63495 : = (final_iter_may_be_partial
11626 64935 : ? wi::udiv_ceil (loop->nb_iterations_likely_upper_bound
11627 1440 : + bias_for_lowest, lowest_vf) - 1
11628 62055 : : wi::udiv_floor (loop->nb_iterations_likely_upper_bound
11629 63495 : + bias_for_lowest, lowest_vf) - 1);
11630 63511 : if (loop->any_estimate)
11631 37071 : loop->nb_iterations_estimate
11632 37071 : = (final_iter_may_be_partial
11633 37772 : ? wi::udiv_ceil (loop->nb_iterations_estimate + bias_for_assumed,
11634 1402 : assumed_vf) - 1
11635 36370 : : wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
11636 73441 : assumed_vf) - 1);
11637 63511 : scale_profile_for_vect_loop (loop_vinfo, loop,
11638 : LOOP_VINFO_MAIN_EXIT (loop_vinfo), assumed_vf,
11639 : flat);
11640 :
11641 63511 : if (dump_enabled_p ())
11642 : {
11643 11339 : if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
11644 : {
11645 9801 : dump_printf_loc (MSG_NOTE, vect_location,
11646 : "LOOP VECTORIZED\n");
11647 9801 : if (loop->inner)
11648 345 : dump_printf_loc (MSG_NOTE, vect_location,
11649 : "OUTER LOOP VECTORIZED\n");
11650 9801 : dump_printf (MSG_NOTE, "\n");
11651 : }
11652 : else
11653 1538 : dump_printf_loc (MSG_NOTE, vect_location,
11654 : "LOOP EPILOGUE VECTORIZED (MODE=%s)\n",
11655 1538 : GET_MODE_NAME (loop_vinfo->vector_mode));
11656 : }
11657 :
11658 : /* Loops vectorized with a variable factor won't benefit from
11659 : unrolling/peeling. */
11660 63511 : if (!vf.is_constant ())
11661 : {
11662 : loop->unroll = 1;
11663 : if (dump_enabled_p ())
11664 : dump_printf_loc (MSG_NOTE, vect_location, "Disabling unrolling due to"
11665 : " variable-length vectorization factor\n");
11666 : }
11667 :
11668 : /* When we have unrolled the loop due to a user requested value we should
11669 : leave it up to the RTL unroll heuristics to determine if it's still worth
11670 : while to unroll more. */
11671 63511 : if (LOOP_VINFO_USER_UNROLL (loop_vinfo))
11672 44 : loop->unroll = 0;
11673 :
11674 : /* Free SLP instances here because otherwise stmt reference counting
11675 : won't work. */
11676 63511 : slp_instance instance;
11677 155845 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
11678 92334 : vect_free_slp_instance (instance);
11679 63511 : LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
11680 : /* Clear-up safelen field since its value is invalid after vectorization
11681 : since vectorized loop can have loop-carried dependencies. */
11682 63511 : loop->safelen = 0;
11683 :
11684 63511 : if (epilogue)
11685 : {
11686 : /* Accumulate past advancements made. */
11687 6862 : if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
11688 91 : advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
11689 : LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
11690 : advance);
11691 6862 : update_epilogue_loop_vinfo (epilogue, advance);
11692 :
11693 6862 : epilogue->simduid = loop->simduid;
11694 6862 : epilogue->force_vectorize = loop->force_vectorize;
11695 6862 : epilogue->dont_vectorize = false;
11696 : }
11697 :
11698 63511 : return epilogue;
11699 63511 : }
11700 :
11701 : /* The code below is trying to perform simple optimization - revert
11702 : if-conversion for masked stores, i.e. if the mask of a store is zero
11703 : do not perform it and all stored value producers also if possible.
11704 : For example,
11705 : for (i=0; i<n; i++)
11706 : if (c[i])
11707 : {
11708 : p1[i] += 1;
11709 : p2[i] = p3[i] +2;
11710 : }
11711 : this transformation will produce the following semi-hammock:
11712 :
11713 : if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
11714 : {
11715 : vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
11716 : vect__12.22_172 = vect__11.19_170 + vect_cst__171;
11717 : MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
11718 : vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
11719 : vect__19.28_184 = vect__18.25_182 + vect_cst__183;
11720 : MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
11721 : }
11722 : */
11723 :
11724 : void
11725 486 : optimize_mask_stores (class loop *loop)
11726 : {
11727 486 : basic_block *bbs = get_loop_body (loop);
11728 486 : unsigned nbbs = loop->num_nodes;
11729 486 : unsigned i;
11730 486 : basic_block bb;
11731 486 : class loop *bb_loop;
11732 486 : gimple_stmt_iterator gsi;
11733 486 : gimple *stmt;
11734 486 : auto_vec<gimple *> worklist;
11735 486 : auto_purge_vect_location sentinel;
11736 :
11737 486 : vect_location = find_loop_location (loop);
11738 : /* Pick up all masked stores in loop if any. */
11739 1944 : for (i = 0; i < nbbs; i++)
11740 : {
11741 972 : bb = bbs[i];
11742 16527 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
11743 14583 : gsi_next (&gsi))
11744 : {
11745 14583 : stmt = gsi_stmt (gsi);
11746 14583 : if (gimple_call_internal_p (stmt, IFN_MASK_STORE))
11747 658 : worklist.safe_push (stmt);
11748 : }
11749 : }
11750 :
11751 486 : free (bbs);
11752 486 : if (worklist.is_empty ())
11753 68 : return;
11754 :
11755 : /* Loop has masked stores. */
11756 1059 : while (!worklist.is_empty ())
11757 : {
11758 641 : gimple *last, *last_store;
11759 641 : edge e, efalse;
11760 641 : tree mask;
11761 641 : basic_block store_bb, join_bb;
11762 641 : gimple_stmt_iterator gsi_to;
11763 641 : tree vdef, new_vdef;
11764 641 : gphi *phi;
11765 641 : tree vectype;
11766 641 : tree zero;
11767 :
11768 641 : last = worklist.pop ();
11769 641 : mask = gimple_call_arg (last, 2);
11770 641 : bb = gimple_bb (last);
11771 : /* Create then_bb and if-then structure in CFG, then_bb belongs to
11772 : the same loop as if_bb. It could be different to LOOP when two
11773 : level loop-nest is vectorized and mask_store belongs to the inner
11774 : one. */
11775 641 : e = split_block (bb, last);
11776 641 : bb_loop = bb->loop_father;
11777 641 : gcc_assert (loop == bb_loop || flow_loop_nested_p (loop, bb_loop));
11778 641 : join_bb = e->dest;
11779 641 : store_bb = create_empty_bb (bb);
11780 641 : add_bb_to_loop (store_bb, bb_loop);
11781 641 : e->flags = EDGE_TRUE_VALUE;
11782 641 : efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
11783 : /* Put STORE_BB to likely part. */
11784 641 : efalse->probability = profile_probability::likely ();
11785 641 : e->probability = efalse->probability.invert ();
11786 641 : store_bb->count = efalse->count ();
11787 641 : make_single_succ_edge (store_bb, join_bb, EDGE_FALLTHRU);
11788 641 : if (dom_info_available_p (CDI_DOMINATORS))
11789 641 : set_immediate_dominator (CDI_DOMINATORS, store_bb, bb);
11790 641 : if (dump_enabled_p ())
11791 326 : dump_printf_loc (MSG_NOTE, vect_location,
11792 : "Create new block %d to sink mask stores.",
11793 : store_bb->index);
11794 : /* Create vector comparison with boolean result. */
11795 641 : vectype = TREE_TYPE (mask);
11796 641 : zero = build_zero_cst (vectype);
11797 641 : stmt = gimple_build_cond (EQ_EXPR, mask, zero, NULL_TREE, NULL_TREE);
11798 641 : gsi = gsi_last_bb (bb);
11799 641 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
11800 : /* Create new PHI node for vdef of the last masked store:
11801 : .MEM_2 = VDEF <.MEM_1>
11802 : will be converted to
11803 : .MEM.3 = VDEF <.MEM_1>
11804 : and new PHI node will be created in join bb
11805 : .MEM_2 = PHI <.MEM_1, .MEM_3>
11806 : */
11807 641 : vdef = gimple_vdef (last);
11808 641 : new_vdef = make_ssa_name (gimple_vop (cfun), last);
11809 641 : gimple_set_vdef (last, new_vdef);
11810 641 : phi = create_phi_node (vdef, join_bb);
11811 641 : add_phi_arg (phi, new_vdef, EDGE_SUCC (store_bb, 0), UNKNOWN_LOCATION);
11812 :
11813 : /* Put all masked stores with the same mask to STORE_BB if possible. */
11814 675 : while (true)
11815 : {
11816 658 : gimple_stmt_iterator gsi_from;
11817 658 : gimple *stmt1 = NULL;
11818 :
11819 : /* Move masked store to STORE_BB. */
11820 658 : last_store = last;
11821 658 : gsi = gsi_for_stmt (last);
11822 658 : gsi_from = gsi;
11823 : /* Shift GSI to the previous stmt for further traversal. */
11824 658 : gsi_prev (&gsi);
11825 658 : gsi_to = gsi_start_bb (store_bb);
11826 658 : gsi_move_before (&gsi_from, &gsi_to);
11827 : /* Setup GSI_TO to the non-empty block start. */
11828 658 : gsi_to = gsi_start_bb (store_bb);
11829 658 : if (dump_enabled_p ())
11830 342 : dump_printf_loc (MSG_NOTE, vect_location,
11831 : "Move stmt to created bb\n%G", last);
11832 : /* Move all stored value producers if possible. */
11833 4929 : while (!gsi_end_p (gsi))
11834 : {
11835 4928 : tree lhs;
11836 4928 : imm_use_iterator imm_iter;
11837 4928 : use_operand_p use_p;
11838 4928 : bool res;
11839 :
11840 : /* Skip debug statements. */
11841 4928 : if (is_gimple_debug (gsi_stmt (gsi)))
11842 : {
11843 1 : gsi_prev (&gsi);
11844 3088 : continue;
11845 : }
11846 4927 : stmt1 = gsi_stmt (gsi);
11847 : /* Do not consider statements writing to memory or having
11848 : volatile operand. */
11849 9679 : if (gimple_vdef (stmt1)
11850 9679 : || gimple_has_volatile_ops (stmt1))
11851 : break;
11852 4752 : gsi_from = gsi;
11853 4752 : gsi_prev (&gsi);
11854 4752 : lhs = gimple_get_lhs (stmt1);
11855 4752 : if (!lhs)
11856 : break;
11857 :
11858 : /* LHS of vectorized stmt must be SSA_NAME. */
11859 4752 : if (TREE_CODE (lhs) != SSA_NAME)
11860 : break;
11861 :
11862 4752 : if (!VECTOR_TYPE_P (TREE_TYPE (lhs)))
11863 : {
11864 : /* Remove dead scalar statement. */
11865 3403 : if (has_zero_uses (lhs))
11866 : {
11867 3087 : gsi_remove (&gsi_from, true);
11868 3087 : release_defs (stmt1);
11869 3087 : continue;
11870 : }
11871 : }
11872 :
11873 : /* Check that LHS does not have uses outside of STORE_BB. */
11874 1665 : res = true;
11875 2872 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
11876 : {
11877 1689 : gimple *use_stmt;
11878 1689 : use_stmt = USE_STMT (use_p);
11879 1689 : if (is_gimple_debug (use_stmt))
11880 0 : continue;
11881 1689 : if (gimple_bb (use_stmt) != store_bb)
11882 : {
11883 : res = false;
11884 : break;
11885 : }
11886 1665 : }
11887 1665 : if (!res)
11888 : break;
11889 :
11890 1183 : if (gimple_vuse (stmt1)
11891 1645 : && gimple_vuse (stmt1) != gimple_vuse (last_store))
11892 : break;
11893 :
11894 : /* Can move STMT1 to STORE_BB. */
11895 1183 : if (dump_enabled_p ())
11896 618 : dump_printf_loc (MSG_NOTE, vect_location,
11897 : "Move stmt to created bb\n%G", stmt1);
11898 1183 : gsi_move_before (&gsi_from, &gsi_to);
11899 : /* Shift GSI_TO for further insertion. */
11900 1183 : gsi_prev (&gsi_to);
11901 : }
11902 : /* Put other masked stores with the same mask to STORE_BB. */
11903 658 : if (worklist.is_empty ()
11904 240 : || gimple_call_arg (worklist.last (), 2) != mask
11905 17 : || worklist.last () != stmt1)
11906 : break;
11907 17 : last = worklist.pop ();
11908 17 : }
11909 1282 : add_phi_arg (phi, gimple_vuse (last_store), e, UNKNOWN_LOCATION);
11910 : }
11911 486 : }
11912 :
11913 : /* Decide whether it is possible to use a zero-based induction variable
11914 : when vectorizing LOOP_VINFO with partial vectors. If it is, return
11915 : the value that the induction variable must be able to hold in order
11916 : to ensure that the rgroups eventually have no active vector elements.
11917 : Return -1 otherwise. */
11918 :
11919 : widest_int
11920 47142 : vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo)
11921 : {
11922 47142 : tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
11923 47142 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
11924 47142 : unsigned HOST_WIDE_INT max_vf = vect_max_vf (loop_vinfo);
11925 :
11926 : /* Calculate the value that the induction variable must be able
11927 : to hit in order to ensure that we end the loop with an all-false mask.
11928 : This involves adding the maximum number of inactive trailing scalar
11929 : iterations. */
11930 47142 : widest_int iv_limit = -1;
11931 47142 : if (max_loop_iterations (loop, &iv_limit))
11932 : {
11933 47142 : if (niters_skip)
11934 : {
11935 : /* Add the maximum number of skipped iterations to the
11936 : maximum iteration count. */
11937 0 : if (TREE_CODE (niters_skip) == INTEGER_CST)
11938 0 : iv_limit += wi::to_widest (niters_skip);
11939 : else
11940 0 : iv_limit += max_vf - 1;
11941 : }
11942 47142 : else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
11943 : /* Make a conservatively-correct assumption. */
11944 326 : iv_limit += max_vf - 1;
11945 :
11946 : /* IV_LIMIT is the maximum number of latch iterations, which is also
11947 : the maximum in-range IV value. Round this value down to the previous
11948 : vector alignment boundary and then add an extra full iteration. */
11949 47142 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
11950 47142 : iv_limit = (iv_limit & -(int) known_alignment (vf)) + max_vf;
11951 : }
11952 47142 : return iv_limit;
11953 : }
11954 :
11955 : /* For the given rgroup_controls RGC, check whether an induction variable
11956 : would ever hit a value that produces a set of all-false masks or zero
11957 : lengths before wrapping around. Return true if it's possible to wrap
11958 : around before hitting the desirable value, otherwise return false. */
11959 :
11960 : bool
11961 0 : vect_rgroup_iv_might_wrap_p (loop_vec_info loop_vinfo, rgroup_controls *rgc)
11962 : {
11963 0 : widest_int iv_limit = vect_iv_limit_for_partial_vectors (loop_vinfo);
11964 :
11965 0 : if (iv_limit == -1)
11966 : return true;
11967 :
11968 0 : tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo);
11969 0 : unsigned int compare_precision = TYPE_PRECISION (compare_type);
11970 0 : unsigned nitems = rgc->max_nscalars_per_iter * rgc->factor;
11971 :
11972 0 : if (wi::min_precision (iv_limit * nitems, UNSIGNED) > compare_precision)
11973 0 : return true;
11974 :
11975 : return false;
11976 0 : }
|
