Line data Source code
1 : /* Statement Analysis and Transformation for Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "ssa.h"
31 : #include "optabs-tree.h"
32 : #include "insn-config.h"
33 : #include "recog.h" /* FIXME: for insn_data */
34 : #include "cgraph.h"
35 : #include "dumpfile.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "tree-eh.h"
40 : #include "gimplify.h"
41 : #include "gimple-iterator.h"
42 : #include "gimplify-me.h"
43 : #include "tree-cfg.h"
44 : #include "tree-ssa-loop-manip.h"
45 : #include "cfgloop.h"
46 : #include "explow.h"
47 : #include "tree-ssa-loop.h"
48 : #include "tree-scalar-evolution.h"
49 : #include "tree-vectorizer.h"
50 : #include "builtins.h"
51 : #include "internal-fn.h"
52 : #include "tree-vector-builder.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "tree-ssa-loop-niter.h"
56 : #include "gimple-fold.h"
57 : #include "regs.h"
58 : #include "attribs.h"
59 : #include "optabs-libfuncs.h"
60 : #include "tree-dfa.h"
61 :
62 : /* For lang_hooks.types.type_for_mode. */
63 : #include "langhooks.h"
64 :
65 : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
66 : bool = false);
67 :
68 : /* Return TRUE iff the given statement is in an inner loop relative to
69 : the loop being vectorized. */
70 : bool
71 6106229 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
72 : {
73 6106229 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
74 6106229 : basic_block bb = gimple_bb (stmt);
75 6106229 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
76 2768332 : class loop* loop;
77 :
78 2768332 : if (!loop_vinfo)
79 : return false;
80 :
81 2768332 : loop = LOOP_VINFO_LOOP (loop_vinfo);
82 :
83 2768332 : return (bb->loop_father == loop->inner);
84 : }
85 :
86 : /* Record the cost of a statement, either by directly informing the
87 : target model or by saving it in a vector for later processing.
88 : Return a preliminary estimate of the statement's cost. */
89 :
90 : unsigned
91 9066624 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
92 : enum vect_cost_for_stmt kind,
93 : stmt_vec_info stmt_info, slp_tree node,
94 : tree vectype, int misalign,
95 : enum vect_cost_model_location where)
96 : {
97 9066624 : if ((kind == vector_load || kind == unaligned_load)
98 1651644 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
99 : kind = vector_gather_load;
100 9066624 : if ((kind == vector_store || kind == unaligned_store)
101 1045210 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
102 9066624 : kind = vector_scatter_store;
103 :
104 9066624 : stmt_info_for_cost si
105 9066624 : = { count, kind, where, stmt_info, node, vectype, misalign };
106 9066624 : body_cost_vec->safe_push (si);
107 :
108 9066624 : return (unsigned)
109 9066624 : (builtin_vectorization_cost (kind, vectype, misalign) * count);
110 : }
111 :
112 : unsigned
113 4226015 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
114 : enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
115 : tree vectype, int misalign,
116 : enum vect_cost_model_location where)
117 : {
118 4226015 : return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
119 4226015 : vectype, misalign, where);
120 : }
121 :
122 : unsigned
123 1704721 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
124 : enum vect_cost_for_stmt kind, slp_tree node,
125 : tree vectype, int misalign,
126 : enum vect_cost_model_location where)
127 : {
128 1704721 : return record_stmt_cost (body_cost_vec, count, kind,
129 : SLP_TREE_REPRESENTATIVE (node), node,
130 1704721 : vectype, misalign, where);
131 : }
132 :
133 : unsigned
134 0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
135 : enum vect_cost_for_stmt kind,
136 : enum vect_cost_model_location where)
137 : {
138 0 : gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
139 : || kind == scalar_stmt);
140 0 : return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
141 0 : NULL_TREE, 0, where);
142 : }
143 :
144 : /* Return a variable of type ELEM_TYPE[NELEMS]. */
145 :
146 : static tree
147 0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
148 : {
149 0 : return create_tmp_var (build_array_type_nelts (elem_type, nelems),
150 0 : "vect_array");
151 : }
152 :
153 : /* ARRAY is an array of vectors created by create_vector_array.
154 : Return an SSA_NAME for the vector in index N. The reference
155 : is part of the vectorization of STMT_INFO and the vector is associated
156 : with scalar destination SCALAR_DEST.
157 : If we need to ensure that inactive elements are set to zero,
158 : NEED_ZEROING is true, MASK contains the loop mask to be used. */
159 :
160 : static tree
161 0 : read_vector_array (vec_info *vinfo,
162 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
163 : tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
164 : bool need_zeroing, tree mask)
165 : {
166 0 : tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
167 0 : gimple *new_stmt;
168 :
169 0 : gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
170 0 : vect_type = TREE_TYPE (TREE_TYPE (array));
171 0 : tmp = vect_create_destination_var (scalar_dest, vect_type);
172 0 : vect = vect_create_destination_var (scalar_dest, vect_type);
173 0 : array_ref = build4 (ARRAY_REF, vect_type, array,
174 0 : build_int_cst (size_type_node, n),
175 : NULL_TREE, NULL_TREE);
176 :
177 0 : new_stmt = gimple_build_assign (tmp, array_ref);
178 0 : tmp_name = make_ssa_name (vect, new_stmt);
179 0 : gimple_assign_set_lhs (new_stmt, tmp_name);
180 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
181 :
182 0 : if (need_zeroing)
183 : {
184 0 : tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
185 : vect_type);
186 0 : vect_name = make_ssa_name (vect, new_stmt);
187 0 : new_stmt
188 0 : = gimple_build_assign (vect_name, VEC_COND_EXPR,
189 : mask, tmp_name, vec_els);
190 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
191 : }
192 : else
193 : vect_name = tmp_name;
194 :
195 0 : return vect_name;
196 : }
197 :
198 : /* ARRAY is an array of vectors created by create_vector_array.
199 : Emit code to store SSA_NAME VECT in index N of the array.
200 : The store is part of the vectorization of STMT_INFO. */
201 :
202 : static void
203 0 : write_vector_array (vec_info *vinfo,
204 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
205 : tree vect, tree array, unsigned HOST_WIDE_INT n)
206 : {
207 0 : tree array_ref;
208 0 : gimple *new_stmt;
209 :
210 0 : array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
211 0 : build_int_cst (size_type_node, n),
212 : NULL_TREE, NULL_TREE);
213 :
214 0 : new_stmt = gimple_build_assign (array_ref, vect);
215 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
216 0 : }
217 :
218 : /* PTR is a pointer to an array of type TYPE. Return a representation
219 : of *PTR. The memory reference replaces those in FIRST_DR
220 : (and its group). */
221 :
222 : static tree
223 0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
224 : {
225 0 : tree mem_ref;
226 :
227 0 : mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
228 : /* Arrays have the same alignment as their type. */
229 0 : set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
230 0 : return mem_ref;
231 : }
232 :
233 : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
234 : Emit the clobber before *GSI. */
235 :
236 : static void
237 15 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
238 : gimple_stmt_iterator *gsi, tree var)
239 : {
240 15 : tree clobber = build_clobber (TREE_TYPE (var));
241 15 : gimple *new_stmt = gimple_build_assign (var, clobber);
242 15 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
243 15 : }
244 :
245 : /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
246 :
247 : /* Function vect_mark_relevant.
248 :
249 : Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
250 :
251 : static void
252 3263492 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
253 : enum vect_relevant relevant, bool live_p)
254 : {
255 3263492 : enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
256 3263492 : bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
257 :
258 3263492 : if (dump_enabled_p ())
259 165005 : dump_printf_loc (MSG_NOTE, vect_location,
260 : "mark relevant %d, live %d: %G", relevant, live_p,
261 : stmt_info->stmt);
262 :
263 : /* If this stmt is an original stmt in a pattern, we might need to mark its
264 : related pattern stmt instead of the original stmt. However, such stmts
265 : may have their own uses that are not in any pattern, in such cases the
266 : stmt itself should be marked. */
267 3263492 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
268 : {
269 : /* This is the last stmt in a sequence that was detected as a
270 : pattern that can potentially be vectorized. Don't mark the stmt
271 : as relevant/live because it's not going to be vectorized.
272 : Instead mark the pattern-stmt that replaces it. */
273 :
274 250913 : if (dump_enabled_p ())
275 2827 : dump_printf_loc (MSG_NOTE, vect_location,
276 : "last stmt in pattern. don't mark"
277 : " relevant/live.\n");
278 :
279 250913 : stmt_vec_info old_stmt_info = stmt_info;
280 250913 : stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
281 250913 : gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
282 250913 : save_relevant = STMT_VINFO_RELEVANT (stmt_info);
283 250913 : save_live_p = STMT_VINFO_LIVE_P (stmt_info);
284 :
285 250913 : if (live_p && relevant == vect_unused_in_scope)
286 : {
287 110 : if (dump_enabled_p ())
288 10 : dump_printf_loc (MSG_NOTE, vect_location,
289 : "vec_stmt_relevant_p: forcing live pattern stmt "
290 : "relevant.\n");
291 : relevant = vect_used_only_live;
292 : }
293 :
294 250913 : if (dump_enabled_p ())
295 2827 : dump_printf_loc (MSG_NOTE, vect_location,
296 : "mark relevant %d, live %d: %G", relevant, live_p,
297 : stmt_info->stmt);
298 : }
299 :
300 3263492 : STMT_VINFO_LIVE_P (stmt_info) |= live_p;
301 3263492 : if (relevant > STMT_VINFO_RELEVANT (stmt_info))
302 2932808 : STMT_VINFO_RELEVANT (stmt_info) = relevant;
303 :
304 3263492 : if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
305 330684 : && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
306 : {
307 329968 : if (dump_enabled_p ())
308 19810 : dump_printf_loc (MSG_NOTE, vect_location,
309 : "already marked relevant/live.\n");
310 : return;
311 : }
312 :
313 2933524 : worklist->safe_push (stmt_info);
314 : }
315 :
316 :
317 : /* Function is_simple_and_all_uses_invariant
318 :
319 : Return true if STMT_INFO is simple and all uses of it are invariant. */
320 :
321 : bool
322 250042 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
323 : loop_vec_info loop_vinfo)
324 : {
325 250042 : tree op;
326 250042 : ssa_op_iter iter;
327 :
328 250042 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
329 194965 : if (!stmt)
330 : return false;
331 :
332 202248 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
333 : {
334 201422 : enum vect_def_type dt = vect_uninitialized_def;
335 :
336 201422 : if (!vect_is_simple_use (op, loop_vinfo, &dt))
337 : {
338 2155 : if (dump_enabled_p ())
339 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
340 : "use not simple.\n");
341 194139 : return false;
342 : }
343 :
344 199267 : if (dt != vect_external_def && dt != vect_constant_def)
345 : return false;
346 : }
347 : return true;
348 : }
349 :
350 : /* Function vect_stmt_relevant_p.
351 :
352 : Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
353 : is "relevant for vectorization".
354 :
355 : A stmt is considered "relevant for vectorization" if:
356 : - it has uses outside the loop.
357 : - it has vdefs (it alters memory).
358 : - control stmts in the loop (except for the exit condition).
359 :
360 : CHECKME: what other side effects would the vectorizer allow? */
361 :
362 : static bool
363 5276890 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
364 : enum vect_relevant *relevant, bool *live_p)
365 : {
366 5276890 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
367 5276890 : ssa_op_iter op_iter;
368 5276890 : imm_use_iterator imm_iter;
369 5276890 : use_operand_p use_p;
370 5276890 : def_operand_p def_p;
371 :
372 5276890 : *relevant = vect_unused_in_scope;
373 5276890 : *live_p = false;
374 :
375 : /* cond stmt other than loop exit cond. */
376 5276890 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
377 5276890 : if (is_ctrl_stmt (stmt)
378 624198 : && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
379 5514439 : && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
380 235566 : *relevant = vect_used_in_scope;
381 :
382 : /* changing memory. */
383 5276890 : if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
384 4370836 : if (gimple_vdef (stmt_info->stmt)
385 3746638 : && !gimple_clobber_p (stmt_info->stmt))
386 : {
387 376317 : if (dump_enabled_p ())
388 28185 : dump_printf_loc (MSG_NOTE, vect_location,
389 : "vec_stmt_relevant_p: stmt has vdefs.\n");
390 376317 : *relevant = vect_used_in_scope;
391 376317 : if (! STMT_VINFO_DATA_REF (stmt_info)
392 376317 : && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
393 20 : LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
394 : }
395 :
396 : /* uses outside the loop. */
397 14823561 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
398 : {
399 11443072 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
400 : {
401 7173291 : basic_block bb = gimple_bb (USE_STMT (use_p));
402 7173291 : if (!flow_bb_inside_loop_p (loop, bb))
403 : {
404 264877 : if (is_gimple_debug (USE_STMT (use_p)))
405 1167 : continue;
406 :
407 263710 : if (dump_enabled_p ())
408 6021 : dump_printf_loc (MSG_NOTE, vect_location,
409 : "vec_stmt_relevant_p: used out of loop.\n");
410 :
411 : /* We expect all such uses to be in the loop exit phis
412 : (because of loop closed form) */
413 263710 : gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
414 :
415 263710 : *live_p = true;
416 263710 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
417 : }
418 4269781 : }
419 : }
420 :
421 250044 : if (*live_p && *relevant == vect_unused_in_scope
422 5526932 : && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
423 : {
424 249216 : if (dump_enabled_p ())
425 5877 : dump_printf_loc (MSG_NOTE, vect_location,
426 : "vec_stmt_relevant_p: stmt live but not relevant.\n");
427 249216 : *relevant = vect_used_only_live;
428 : }
429 :
430 5276890 : return (*live_p || *relevant);
431 : }
432 :
433 :
434 : /* Function exist_non_indexing_operands_for_use_p
435 :
436 : USE is one of the uses attached to STMT_INFO. Check if USE is
437 : used in STMT_INFO for anything other than indexing an array. */
438 :
439 : static bool
440 4374313 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
441 : {
442 4374313 : tree operand;
443 :
444 : /* USE corresponds to some operand in STMT. If there is no data
445 : reference in STMT, then any operand that corresponds to USE
446 : is not indexing an array. */
447 4374313 : if (!STMT_VINFO_DATA_REF (stmt_info))
448 : return true;
449 :
450 : /* STMT has a data_ref. FORNOW this means that its of one of
451 : the following forms:
452 : -1- ARRAY_REF = var
453 : -2- var = ARRAY_REF
454 : (This should have been verified in analyze_data_refs).
455 :
456 : 'var' in the second case corresponds to a def, not a use,
457 : so USE cannot correspond to any operands that are not used
458 : for array indexing.
459 :
460 : Therefore, all we need to check is if STMT falls into the
461 : first case, and whether var corresponds to USE. */
462 :
463 1492566 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
464 1474580 : if (!assign || !gimple_assign_copy_p (assign))
465 : {
466 799826 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
467 17986 : if (call && gimple_call_internal_p (call))
468 : {
469 17986 : internal_fn ifn = gimple_call_internal_fn (call);
470 17986 : int mask_index = internal_fn_mask_index (ifn);
471 17986 : if (mask_index >= 0
472 17986 : && use == gimple_call_arg (call, mask_index))
473 : return true;
474 11670 : int els_index = internal_fn_else_index (ifn);
475 11670 : if (els_index >= 0
476 11670 : && use == gimple_call_arg (call, els_index))
477 : return true;
478 10166 : int stored_value_index = internal_fn_stored_value_index (ifn);
479 10166 : if (stored_value_index >= 0
480 10166 : && use == gimple_call_arg (call, stored_value_index))
481 : return true;
482 8016 : if (internal_gather_scatter_fn_p (ifn)
483 8016 : && use == gimple_call_arg (call, 1))
484 : return true;
485 : }
486 789856 : return false;
487 : }
488 :
489 692740 : if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
490 : return false;
491 692740 : operand = gimple_assign_rhs1 (assign);
492 692740 : if (TREE_CODE (operand) != SSA_NAME)
493 : return false;
494 :
495 599050 : if (operand == use)
496 282806 : return true;
497 :
498 : return false;
499 : }
500 :
501 :
502 : /*
503 : Function process_use.
504 :
505 : Inputs:
506 : - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
507 : - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
508 : that defined USE. This is done by calling mark_relevant and passing it
509 : the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
510 : - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
511 : be performed.
512 :
513 : Outputs:
514 : Generally, LIVE_P and RELEVANT are used to define the liveness and
515 : relevance info of the DEF_STMT of this USE:
516 : STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
517 : STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
518 : Exceptions:
519 : - case 1: If USE is used only for address computations (e.g. array indexing),
520 : which does not need to be directly vectorized, then the liveness/relevance
521 : of the respective DEF_STMT is left unchanged.
522 : - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
523 : we skip DEF_STMT cause it had already been processed.
524 : - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
525 : "relevant" will be modified accordingly.
526 :
527 : Return true if everything is as expected. Return false otherwise. */
528 :
529 : static opt_result
530 4432231 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
531 : enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
532 : bool force)
533 : {
534 4432231 : stmt_vec_info dstmt_vinfo;
535 4432231 : enum vect_def_type dt;
536 :
537 : /* case 1: we are only interested in uses that need to be vectorized. Uses
538 : that are used for address computation are not considered relevant. */
539 4432231 : if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
540 1199790 : return opt_result::success ();
541 :
542 3232441 : if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
543 35152 : return opt_result::failure_at (stmt_vinfo->stmt,
544 : "not vectorized:"
545 : " unsupported use in stmt.\n");
546 :
547 3197289 : if (!dstmt_vinfo)
548 603704 : return opt_result::success ();
549 :
550 2593585 : basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
551 2593585 : basic_block bb = gimple_bb (stmt_vinfo->stmt);
552 :
553 : /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
554 : We have to force the stmt live since the epilogue loop needs it to
555 : continue computing the reduction. */
556 2593585 : if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
557 273826 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
558 85854 : && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
559 85854 : && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
560 2679439 : && bb->loop_father == def_bb->loop_father)
561 : {
562 85854 : if (dump_enabled_p ())
563 3960 : dump_printf_loc (MSG_NOTE, vect_location,
564 : "reduc-stmt defining reduc-phi in the same nest.\n");
565 85854 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
566 85854 : return opt_result::success ();
567 : }
568 :
569 : /* case 3a: outer-loop stmt defining an inner-loop stmt:
570 : outer-loop-header-bb:
571 : d = dstmt_vinfo
572 : inner-loop:
573 : stmt # use (d)
574 : outer-loop-tail-bb:
575 : ... */
576 2507731 : if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
577 : {
578 2238 : if (dump_enabled_p ())
579 321 : dump_printf_loc (MSG_NOTE, vect_location,
580 : "outer-loop def-stmt defining inner-loop stmt.\n");
581 :
582 2238 : switch (relevant)
583 : {
584 0 : case vect_unused_in_scope:
585 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
586 : vect_used_in_scope : vect_unused_in_scope;
587 : break;
588 :
589 779 : case vect_used_in_outer_by_reduction:
590 779 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
591 : relevant = vect_used_by_reduction;
592 : break;
593 :
594 1179 : case vect_used_in_outer:
595 1179 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
596 : relevant = vect_used_in_scope;
597 : break;
598 :
599 : case vect_used_in_scope:
600 : break;
601 :
602 0 : default:
603 0 : gcc_unreachable ();
604 : }
605 : }
606 :
607 : /* case 3b: inner-loop stmt defining an outer-loop stmt:
608 : outer-loop-header-bb:
609 : ...
610 : inner-loop:
611 : d = dstmt_vinfo
612 : outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
613 : stmt # use (d) */
614 2505493 : else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
615 : {
616 2098 : if (dump_enabled_p ())
617 626 : dump_printf_loc (MSG_NOTE, vect_location,
618 : "inner-loop def-stmt defining outer-loop stmt.\n");
619 :
620 2098 : switch (relevant)
621 : {
622 0 : case vect_unused_in_scope:
623 2326132 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
624 0 : || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
625 : vect_used_in_outer_by_reduction : vect_unused_in_scope;
626 : break;
627 :
628 : case vect_used_by_reduction:
629 : case vect_used_only_live:
630 : relevant = vect_used_in_outer_by_reduction;
631 : break;
632 :
633 : case vect_used_in_scope:
634 2326132 : relevant = vect_used_in_outer;
635 : break;
636 :
637 0 : default:
638 0 : gcc_unreachable ();
639 : }
640 : }
641 : /* We are also not interested in uses on loop PHI backedges that are
642 : inductions. Otherwise we'll needlessly vectorize the IV increment
643 : and cause hybrid SLP for SLP inductions. */
644 2503395 : else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
645 184642 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
646 2684994 : && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
647 : loop_latch_edge (bb->loop_father))
648 : == use))
649 : {
650 181599 : if (dump_enabled_p ())
651 4882 : dump_printf_loc (MSG_NOTE, vect_location,
652 : "induction value on backedge.\n");
653 181599 : return opt_result::success ();
654 : }
655 :
656 2326132 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
657 2326132 : return opt_result::success ();
658 : }
659 :
660 :
661 : /* Function vect_mark_stmts_to_be_vectorized.
662 :
663 : Not all stmts in the loop need to be vectorized. For example:
664 :
665 : for i...
666 : for j...
667 : 1. T0 = i + j
668 : 2. T1 = a[T0]
669 :
670 : 3. j = j + 1
671 :
672 : Stmt 1 and 3 do not need to be vectorized, because loop control and
673 : addressing of vectorized data-refs are handled differently.
674 :
675 : This pass detects such stmts. */
676 :
677 : opt_result
678 444088 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
679 : {
680 444088 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
681 444088 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
682 444088 : unsigned int nbbs = loop->num_nodes;
683 444088 : gimple_stmt_iterator si;
684 444088 : unsigned int i;
685 444088 : basic_block bb;
686 444088 : bool live_p;
687 444088 : enum vect_relevant relevant;
688 :
689 444088 : DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
690 :
691 444088 : auto_vec<stmt_vec_info, 64> worklist;
692 :
693 : /* 1. Init worklist. */
694 1505800 : for (i = 0; i < nbbs; i++)
695 : {
696 1072276 : bb = bbs[i];
697 2202346 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
698 : {
699 2280978 : if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
700 234435 : continue;
701 906054 : stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
702 906054 : if (dump_enabled_p ())
703 41794 : dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
704 : phi_info->stmt);
705 :
706 906054 : if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
707 : {
708 46206 : if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
709 10419 : return opt_result::failure_at
710 10419 : (*si, "not vectorized: unhandled relevant PHI: %G", *si);
711 35787 : vect_mark_relevant (&worklist, phi_info, relevant, live_p);
712 : }
713 : }
714 8546462 : for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
715 : {
716 7484750 : gimple *stmt = gsi_stmt (si);
717 7484750 : if (is_gimple_debug (stmt))
718 3113769 : continue;
719 4370981 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
720 4370981 : if (dump_enabled_p ())
721 224529 : dump_printf_loc (MSG_NOTE, vect_location,
722 : "init: stmt relevant? %G", stmt);
723 :
724 4370981 : if (gimple_get_lhs (stmt) == NULL_TREE
725 630426 : && !is_a <gcond *> (stmt)
726 4377209 : && !is_a <gcall *> (stmt))
727 145 : return opt_result::failure_at
728 145 : (stmt, "not vectorized: irregular stmt: %G", stmt);
729 :
730 4370836 : if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
731 815719 : vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
732 : }
733 : }
734 :
735 : /* 2. Process_worklist */
736 3238005 : while (worklist.length () > 0)
737 : {
738 2839635 : use_operand_p use_p;
739 2839635 : ssa_op_iter iter;
740 :
741 2839635 : stmt_vec_info stmt_vinfo = worklist.pop ();
742 2839635 : if (dump_enabled_p ())
743 144552 : dump_printf_loc (MSG_NOTE, vect_location,
744 : "worklist: examine stmt: %G", stmt_vinfo->stmt);
745 :
746 : /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
747 : (DEF_STMT) as relevant/irrelevant according to the relevance property
748 : of STMT. */
749 2839635 : relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
750 :
751 : /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
752 : propagated as is to the DEF_STMTs of its USEs.
753 :
754 : One exception is when STMT has been identified as defining a reduction
755 : variable; in this case we set the relevance to vect_used_by_reduction.
756 : This is because we distinguish between two kinds of relevant stmts -
757 : those that are used by a reduction computation, and those that are
758 : (also) used by a regular computation. This allows us later on to
759 : identify stmts that are used solely by a reduction, and therefore the
760 : order of the results that they produce does not have to be kept. */
761 :
762 2839635 : switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
763 : {
764 174651 : case vect_reduction_def:
765 174651 : gcc_assert (relevant != vect_unused_in_scope);
766 174651 : if (relevant != vect_unused_in_scope
767 174651 : && relevant != vect_used_in_scope
768 174651 : && relevant != vect_used_by_reduction
769 174651 : && relevant != vect_used_only_live)
770 0 : return opt_result::failure_at
771 0 : (stmt_vinfo->stmt, "unsupported use of reduction.\n");
772 : break;
773 :
774 2209 : case vect_nested_cycle:
775 2209 : if (relevant != vect_unused_in_scope
776 2209 : && relevant != vect_used_in_outer_by_reduction
777 1614 : && relevant != vect_used_in_outer)
778 2 : return opt_result::failure_at
779 2 : (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
780 : break;
781 :
782 1209 : case vect_double_reduction_def:
783 1209 : if (relevant != vect_unused_in_scope
784 1209 : && relevant != vect_used_by_reduction
785 409 : && relevant != vect_used_only_live)
786 0 : return opt_result::failure_at
787 0 : (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
788 : break;
789 :
790 : default:
791 : break;
792 : }
793 :
794 2839633 : if (is_pattern_stmt_p (stmt_vinfo))
795 : {
796 : /* Pattern statements are not inserted into the code, so
797 : FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
798 : have to scan the RHS or function arguments instead. */
799 641114 : if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
800 : {
801 415934 : enum tree_code rhs_code = gimple_assign_rhs_code (assign);
802 415934 : tree op = gimple_assign_rhs1 (assign);
803 :
804 415934 : i = 1;
805 415934 : if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
806 : {
807 0 : opt_result res
808 0 : = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
809 : loop_vinfo, relevant, &worklist, false);
810 0 : if (!res)
811 0 : return res;
812 0 : res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
813 : loop_vinfo, relevant, &worklist, false);
814 0 : if (!res)
815 0 : return res;
816 415934 : i = 2;
817 : }
818 1197567 : for (; i < gimple_num_ops (assign); i++)
819 : {
820 785488 : op = gimple_op (assign, i);
821 785488 : if (TREE_CODE (op) == SSA_NAME)
822 : {
823 595990 : opt_result res
824 595990 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
825 : &worklist, false);
826 595990 : if (!res)
827 3855 : return res;
828 : }
829 : }
830 : }
831 225180 : else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
832 : {
833 218563 : tree_code rhs_code = gimple_cond_code (cond);
834 218563 : gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
835 218563 : opt_result res
836 218563 : = process_use (stmt_vinfo, gimple_cond_lhs (cond),
837 : loop_vinfo, relevant, &worklist, false);
838 218563 : if (!res)
839 35154 : return res;
840 218563 : res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
841 : loop_vinfo, relevant, &worklist, false);
842 218563 : if (!res)
843 0 : return res;
844 : }
845 6617 : else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
846 : {
847 31637 : for (i = 0; i < gimple_call_num_args (call); i++)
848 : {
849 25020 : tree arg = gimple_call_arg (call, i);
850 25020 : opt_result res
851 25020 : = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
852 : &worklist, false);
853 25020 : if (!res)
854 0 : return res;
855 : }
856 : }
857 : else
858 0 : gcc_unreachable ();
859 : }
860 : else
861 7695203 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
862 : {
863 3316177 : tree op = USE_FROM_PTR (use_p);
864 3316177 : opt_result res
865 3316177 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
866 : &worklist, false);
867 3316177 : if (!res)
868 18012 : return res;
869 : }
870 :
871 2817766 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
872 : {
873 57918 : gather_scatter_info gs_info;
874 57918 : if (!vect_check_gather_scatter (stmt_vinfo,
875 : STMT_VINFO_VECTYPE (stmt_vinfo),
876 : loop_vinfo, &gs_info))
877 0 : gcc_unreachable ();
878 57918 : opt_result res
879 57918 : = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
880 : &worklist, true);
881 57918 : if (!res)
882 : {
883 13285 : if (fatal)
884 13285 : *fatal = false;
885 13285 : return res;
886 : }
887 : }
888 : } /* while worklist */
889 :
890 398370 : return opt_result::success ();
891 444088 : }
892 :
893 : /* Function vect_model_simple_cost.
894 :
895 : Models cost for simple operations, i.e. those that only emit N operations
896 : of the same KIND. */
897 :
898 : static void
899 668295 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
900 : stmt_vector_for_cost *cost_vec,
901 : vect_cost_for_stmt kind = vector_stmt)
902 : {
903 668295 : int inside_cost = 0, prologue_cost = 0;
904 :
905 668295 : gcc_assert (cost_vec != NULL);
906 :
907 668295 : n *= vect_get_num_copies (vinfo, node);
908 :
909 : /* Pass the inside-of-loop statements to the target-specific cost model. */
910 668295 : inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
911 :
912 668295 : if (dump_enabled_p ())
913 33431 : dump_printf_loc (MSG_NOTE, vect_location,
914 : "vect_model_simple_cost: inside_cost = %d, "
915 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
916 668295 : }
917 :
918 :
919 : /* Model cost for type demotion and promotion operations. PWR is
920 : normally zero for single-step promotions and demotions. It will be
921 : one if two-step promotion/demotion is required, and so on. NCOPIES
922 : is the number of vector results (and thus number of instructions)
923 : for the narrowest end of the operation chain. Each additional
924 : step doubles the number of instructions required. If WIDEN_ARITH
925 : is true the stmt is doing widening arithmetic. */
926 :
927 : static void
928 69432 : vect_model_promotion_demotion_cost (slp_tree slp_node,
929 : unsigned int ncopies, int pwr,
930 : stmt_vector_for_cost *cost_vec,
931 : bool widen_arith)
932 : {
933 69432 : int i;
934 69432 : int inside_cost = 0, prologue_cost = 0;
935 :
936 161702 : for (i = 0; i < pwr + 1; i++)
937 : {
938 182619 : inside_cost += record_stmt_cost (cost_vec, ncopies,
939 : widen_arith
940 : ? vector_stmt : vec_promote_demote,
941 : slp_node, 0, vect_body);
942 92270 : ncopies *= 2;
943 : }
944 :
945 69432 : if (dump_enabled_p ())
946 6380 : dump_printf_loc (MSG_NOTE, vect_location,
947 : "vect_model_promotion_demotion_cost: inside_cost = %d, "
948 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
949 69432 : }
950 :
951 : /* Returns true if the current function returns DECL. */
952 :
953 : static bool
954 569696 : cfun_returns (tree decl)
955 : {
956 569696 : edge_iterator ei;
957 569696 : edge e;
958 1121866 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
959 : {
960 1128040 : greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
961 564020 : if (!ret)
962 0 : continue;
963 564020 : if (gimple_return_retval (ret) == decl)
964 : return true;
965 : /* We often end up with an aggregate copy to the result decl,
966 : handle that case as well. First skip intermediate clobbers
967 : though. */
968 : gimple *def = ret;
969 1706557 : do
970 : {
971 3413114 : def = SSA_NAME_DEF_STMT (gimple_vuse (def));
972 : }
973 1706557 : while (gimple_clobber_p (def));
974 552975 : if (is_a <gassign *> (def)
975 62841 : && gimple_assign_lhs (def) == gimple_return_retval (ret)
976 560198 : && gimple_assign_rhs1 (def) == decl)
977 : return true;
978 : }
979 : return false;
980 : }
981 :
982 : /* Calculate cost of DR's memory access. */
983 : void
984 1035135 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
985 : int ncopies, dr_alignment_support alignment_support_scheme,
986 : int misalignment,
987 : unsigned int *inside_cost,
988 : stmt_vector_for_cost *body_cost_vec)
989 : {
990 1035135 : tree vectype
991 1035135 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
992 1035135 : switch (alignment_support_scheme)
993 : {
994 568954 : case dr_aligned:
995 568954 : {
996 568954 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
997 : vector_store, stmt_info, slp_node,
998 : vectype, 0, vect_body);
999 :
1000 568954 : if (dump_enabled_p ())
1001 14570 : dump_printf_loc (MSG_NOTE, vect_location,
1002 : "vect_model_store_cost: aligned.\n");
1003 : break;
1004 : }
1005 :
1006 466181 : case dr_unaligned_supported:
1007 466181 : {
1008 : /* Here, we assign an additional cost for the unaligned store. */
1009 466181 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1010 : unaligned_store, stmt_info, slp_node,
1011 : vectype, misalignment, vect_body);
1012 466181 : if (dump_enabled_p ())
1013 13013 : dump_printf_loc (MSG_NOTE, vect_location,
1014 : "vect_model_store_cost: unaligned supported by "
1015 : "hardware.\n");
1016 : break;
1017 : }
1018 :
1019 0 : case dr_unaligned_unsupported:
1020 0 : {
1021 0 : *inside_cost = VECT_MAX_COST;
1022 :
1023 0 : if (dump_enabled_p ())
1024 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1025 : "vect_model_store_cost: unsupported access.\n");
1026 : break;
1027 : }
1028 :
1029 0 : default:
1030 0 : gcc_unreachable ();
1031 : }
1032 1035135 : }
1033 :
1034 : /* Calculate cost of DR's memory access. */
1035 : void
1036 977763 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
1037 : int ncopies, dr_alignment_support alignment_support_scheme,
1038 : int misalignment,
1039 : bool add_realign_cost, unsigned int *inside_cost,
1040 : unsigned int *prologue_cost,
1041 : stmt_vector_for_cost *prologue_cost_vec,
1042 : stmt_vector_for_cost *body_cost_vec,
1043 : bool record_prologue_costs)
1044 : {
1045 977763 : tree vectype
1046 977763 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
1047 977763 : switch (alignment_support_scheme)
1048 : {
1049 562087 : case dr_aligned:
1050 562087 : {
1051 562087 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1052 : stmt_info, slp_node, vectype,
1053 : 0, vect_body);
1054 :
1055 562087 : if (dump_enabled_p ())
1056 19133 : dump_printf_loc (MSG_NOTE, vect_location,
1057 : "vect_model_load_cost: aligned.\n");
1058 :
1059 : break;
1060 : }
1061 355114 : case dr_unaligned_supported:
1062 355114 : {
1063 : /* Here, we assign an additional cost for the unaligned load. */
1064 355114 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1065 : unaligned_load, stmt_info, slp_node,
1066 : vectype, misalignment, vect_body);
1067 :
1068 355114 : if (dump_enabled_p ())
1069 22497 : dump_printf_loc (MSG_NOTE, vect_location,
1070 : "vect_model_load_cost: unaligned supported by "
1071 : "hardware.\n");
1072 :
1073 : break;
1074 : }
1075 0 : case dr_explicit_realign:
1076 0 : {
1077 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1078 : vector_load, stmt_info, slp_node,
1079 : vectype, 0, vect_body);
1080 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1081 : vec_perm, stmt_info, slp_node,
1082 : vectype, 0, vect_body);
1083 :
1084 : /* FIXME: If the misalignment remains fixed across the iterations of
1085 : the containing loop, the following cost should be added to the
1086 : prologue costs. */
1087 0 : if (targetm.vectorize.builtin_mask_for_load)
1088 0 : *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1089 : stmt_info, slp_node, vectype,
1090 : 0, vect_body);
1091 :
1092 0 : if (dump_enabled_p ())
1093 0 : dump_printf_loc (MSG_NOTE, vect_location,
1094 : "vect_model_load_cost: explicit realign\n");
1095 :
1096 : break;
1097 : }
1098 0 : case dr_explicit_realign_optimized:
1099 0 : {
1100 0 : if (dump_enabled_p ())
1101 0 : dump_printf_loc (MSG_NOTE, vect_location,
1102 : "vect_model_load_cost: unaligned software "
1103 : "pipelined.\n");
1104 :
1105 : /* Unaligned software pipeline has a load of an address, an initial
1106 : load, and possibly a mask operation to "prime" the loop. However,
1107 : if this is an access in a group of loads, which provide grouped
1108 : access, then the above cost should only be considered for one
1109 : access in the group. Inside the loop, there is a load op
1110 : and a realignment op. */
1111 :
1112 0 : if (add_realign_cost && record_prologue_costs)
1113 : {
1114 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1115 : vector_stmt, stmt_info,
1116 : slp_node, vectype,
1117 : 0, vect_prologue);
1118 0 : if (targetm.vectorize.builtin_mask_for_load)
1119 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1120 : vector_stmt, stmt_info,
1121 : slp_node, vectype,
1122 : 0, vect_prologue);
1123 : }
1124 :
1125 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1126 : stmt_info, slp_node, vectype,
1127 : 0, vect_body);
1128 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1129 : stmt_info, slp_node, vectype,
1130 : 0, vect_body);
1131 :
1132 0 : if (dump_enabled_p ())
1133 0 : dump_printf_loc (MSG_NOTE, vect_location,
1134 : "vect_model_load_cost: explicit realign optimized"
1135 : "\n");
1136 :
1137 : break;
1138 : }
1139 :
1140 60562 : case dr_unaligned_unsupported:
1141 60562 : {
1142 60562 : *inside_cost = VECT_MAX_COST;
1143 :
1144 60562 : if (dump_enabled_p ())
1145 104 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1146 : "vect_model_load_cost: unsupported access.\n");
1147 : break;
1148 : }
1149 :
1150 0 : default:
1151 0 : gcc_unreachable ();
1152 : }
1153 977763 : }
1154 :
1155 : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1156 : the loop preheader for the vectorized stmt STMT_VINFO. */
1157 :
1158 : static void
1159 6664 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
1160 : gimple_stmt_iterator *gsi)
1161 : {
1162 6664 : if (gsi)
1163 3371 : vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
1164 : else
1165 3293 : vinfo->insert_on_entry (stmt_vinfo, new_stmt);
1166 :
1167 6664 : if (dump_enabled_p ())
1168 1820 : dump_printf_loc (MSG_NOTE, vect_location,
1169 : "created new init_stmt: %G", new_stmt);
1170 6664 : }
1171 :
1172 : /* Function vect_init_vector.
1173 :
1174 : Insert a new stmt (INIT_STMT) that initializes a new variable of type
1175 : TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1176 : vector type a vector with all elements equal to VAL is created first.
1177 : Place the initialization at GSI if it is not NULL. Otherwise, place the
1178 : initialization at the loop preheader.
1179 : Return the DEF of INIT_STMT.
1180 : It will be used in the vectorization of STMT_INFO. */
1181 :
1182 : tree
1183 4947 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
1184 : gimple_stmt_iterator *gsi)
1185 : {
1186 4947 : gimple *init_stmt;
1187 4947 : tree new_temp;
1188 :
1189 : /* We abuse this function to push sth to a SSA name with initial 'val'. */
1190 4947 : if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1191 : {
1192 1336 : gcc_assert (VECTOR_TYPE_P (type));
1193 1336 : if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1194 : {
1195 : /* Scalar boolean value should be transformed into
1196 : all zeros or all ones value before building a vector. */
1197 11 : if (VECTOR_BOOLEAN_TYPE_P (type))
1198 : {
1199 3 : tree true_val = build_all_ones_cst (TREE_TYPE (type));
1200 3 : tree false_val = build_zero_cst (TREE_TYPE (type));
1201 :
1202 3 : if (CONSTANT_CLASS_P (val))
1203 0 : val = integer_zerop (val) ? false_val : true_val;
1204 : else
1205 : {
1206 3 : new_temp = make_ssa_name (TREE_TYPE (type));
1207 3 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1208 : val, true_val, false_val);
1209 3 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1210 3 : val = new_temp;
1211 : }
1212 : }
1213 : else
1214 : {
1215 8 : gimple_seq stmts = NULL;
1216 8 : if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1217 8 : val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
1218 8 : TREE_TYPE (type), val);
1219 : else
1220 : /* ??? Condition vectorization expects us to do
1221 : promotion of invariant/external defs. */
1222 0 : val = gimple_convert (&stmts, TREE_TYPE (type), val);
1223 16 : for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
1224 16 : !gsi_end_p (gsi2); )
1225 : {
1226 8 : init_stmt = gsi_stmt (gsi2);
1227 8 : gsi_remove (&gsi2, false);
1228 8 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1229 : }
1230 : }
1231 : }
1232 1336 : val = build_vector_from_val (type, val);
1233 : }
1234 :
1235 4947 : new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1236 4947 : init_stmt = gimple_build_assign (new_temp, val);
1237 4947 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1238 4947 : return new_temp;
1239 : }
1240 :
1241 :
1242 : /* Get vectorized definitions for OP0 and OP1. */
1243 :
1244 : void
1245 191501 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
1246 : bool op0, vec<tree> *vec_oprnds0,
1247 : bool op1, vec<tree> *vec_oprnds1,
1248 : bool op2, vec<tree> *vec_oprnds2,
1249 : bool op3, vec<tree> *vec_oprnds3)
1250 : {
1251 191501 : if (op0)
1252 189839 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
1253 191501 : if (op1)
1254 140635 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
1255 191501 : if (op2)
1256 9382 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
1257 191501 : if (op3)
1258 0 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
1259 191501 : }
1260 :
1261 : /* Helper function called by vect_finish_replace_stmt and
1262 : vect_finish_stmt_generation. Set the location of the new
1263 : statement and create and return a stmt_vec_info for it. */
1264 :
1265 : static void
1266 1454642 : vect_finish_stmt_generation_1 (vec_info *,
1267 : stmt_vec_info stmt_info, gimple *vec_stmt)
1268 : {
1269 1454642 : if (dump_enabled_p ())
1270 147706 : dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
1271 :
1272 1454642 : if (stmt_info)
1273 : {
1274 1426088 : gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
1275 :
1276 : /* While EH edges will generally prevent vectorization, stmt might
1277 : e.g. be in a must-not-throw region. Ensure newly created stmts
1278 : that could throw are part of the same region. */
1279 1426088 : int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
1280 1426088 : if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
1281 48 : add_stmt_to_eh_lp (vec_stmt, lp_nr);
1282 : }
1283 : else
1284 28554 : gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
1285 1454642 : }
1286 :
1287 : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1288 : which sets the same scalar result as STMT_INFO did. Create and return a
1289 : stmt_vec_info for VEC_STMT. */
1290 :
1291 : void
1292 913 : vect_finish_replace_stmt (vec_info *vinfo,
1293 : stmt_vec_info stmt_info, gimple *vec_stmt)
1294 : {
1295 913 : gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
1296 913 : gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
1297 :
1298 913 : gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
1299 913 : gsi_replace (&gsi, vec_stmt, true);
1300 :
1301 913 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1302 913 : }
1303 :
1304 : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1305 : before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1306 :
1307 : void
1308 1453729 : vect_finish_stmt_generation (vec_info *vinfo,
1309 : stmt_vec_info stmt_info, gimple *vec_stmt,
1310 : gimple_stmt_iterator *gsi)
1311 : {
1312 1453729 : gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
1313 :
1314 1453729 : if (!gsi_end_p (*gsi)
1315 2906329 : && gimple_has_mem_ops (vec_stmt))
1316 : {
1317 1452600 : gimple *at_stmt = gsi_stmt (*gsi);
1318 1452600 : tree vuse = gimple_vuse (at_stmt);
1319 1446569 : if (vuse && TREE_CODE (vuse) == SSA_NAME)
1320 : {
1321 1302305 : tree vdef = gimple_vdef (at_stmt);
1322 1302305 : gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1323 1302305 : gimple_set_modified (vec_stmt, true);
1324 : /* If we have an SSA vuse and insert a store, update virtual
1325 : SSA form to avoid triggering the renamer. Do so only
1326 : if we can easily see all uses - which is what almost always
1327 : happens with the way vectorized stmts are inserted. */
1328 764370 : if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1329 2066639 : && ((is_gimple_assign (vec_stmt)
1330 763497 : && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1331 64050 : || (is_gimple_call (vec_stmt)
1332 837 : && (!(gimple_call_flags (vec_stmt)
1333 837 : & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
1334 3 : || (gimple_call_lhs (vec_stmt)
1335 3 : && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
1336 : {
1337 701118 : tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1338 701118 : gimple_set_vdef (vec_stmt, new_vdef);
1339 701118 : SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1340 : }
1341 : }
1342 : }
1343 1453729 : gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1344 1453729 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1345 1453729 : }
1346 :
1347 : /* We want to vectorize a call to combined function CFN with function
1348 : decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1349 : as the types of all inputs. Check whether this is possible using
1350 : an internal function, returning its code if so or IFN_LAST if not. */
1351 :
1352 : static internal_fn
1353 16341 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
1354 : tree vectype_out, tree vectype_in)
1355 : {
1356 16341 : internal_fn ifn;
1357 16341 : if (internal_fn_p (cfn))
1358 13819 : ifn = as_internal_fn (cfn);
1359 : else
1360 2522 : ifn = associated_internal_fn (fndecl);
1361 16341 : if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1362 : {
1363 12939 : const direct_internal_fn_info &info = direct_internal_fn (ifn);
1364 12939 : if (info.vectorizable)
1365 : {
1366 12939 : bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
1367 12939 : tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1368 12939 : tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1369 :
1370 : /* The type size of both the vectype_in and vectype_out should be
1371 : exactly the same when vectype_out isn't participating the optab.
1372 : While there is no restriction for type size when vectype_out
1373 : is part of the optab query. */
1374 12939 : if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
1375 16341 : return IFN_LAST;
1376 :
1377 12902 : if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1378 : OPTIMIZE_FOR_SPEED))
1379 7469 : return ifn;
1380 : }
1381 : }
1382 : return IFN_LAST;
1383 : }
1384 :
1385 :
1386 : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
1387 : gimple_stmt_iterator *);
1388 :
1389 : /* Check whether a load or store statement in the loop described by
1390 : LOOP_VINFO is possible in a loop using partial vectors. This is
1391 : testing whether the vectorizer pass has the appropriate support,
1392 : as well as whether the target does.
1393 :
1394 : VLS_TYPE says whether the statement is a load or store and VECTYPE
1395 : is the type of the vector being loaded or stored. SLP_NODE is the SLP
1396 : node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1397 : says how the load or store is going to be implemented and GROUP_SIZE
1398 : is the number of load or store statements in the containing group.
1399 : If the access is a gather load or scatter store, GS_INFO describes
1400 : its arguments. If the load or store is conditional, SCALAR_MASK is the
1401 : condition under which it occurs.
1402 :
1403 : Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1404 : vectors is not supported, otherwise record the required rgroup control
1405 : types.
1406 :
1407 : If partial vectors can be used and ELSVALS is nonzero the supported
1408 : else values will be added to the vector ELSVALS points to. */
1409 :
1410 : static void
1411 296982 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
1412 : slp_tree slp_node,
1413 : vec_load_store_type vls_type,
1414 : int group_size,
1415 : vect_load_store_data *ls,
1416 : slp_tree mask_node,
1417 : vec<int> *elsvals = nullptr)
1418 : {
1419 296982 : vect_memory_access_type memory_access_type = ls->memory_access_type;
1420 :
1421 : /* Invariant loads need no special support. */
1422 296982 : if (memory_access_type == VMAT_INVARIANT)
1423 29455 : return;
1424 :
1425 : /* Figure whether the mask is uniform. scalar_mask is used to
1426 : populate the scalar_cond_masked_set. */
1427 295794 : tree scalar_mask = NULL_TREE;
1428 295794 : if (mask_node)
1429 4906 : for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
1430 : {
1431 2484 : tree def = vect_get_slp_scalar_def (mask_node, i);
1432 2484 : if (!def
1433 2484 : || (scalar_mask && def != scalar_mask))
1434 : {
1435 : scalar_mask = NULL;
1436 : break;
1437 : }
1438 : else
1439 2463 : scalar_mask = def;
1440 : }
1441 :
1442 295794 : unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
1443 295794 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1444 295794 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
1445 295794 : machine_mode vecmode = TYPE_MODE (vectype);
1446 295794 : bool is_load = (vls_type == VLS_LOAD);
1447 295794 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
1448 : {
1449 0 : nvectors /= group_size;
1450 0 : internal_fn ifn
1451 0 : = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
1452 : elsvals)
1453 0 : : vect_store_lanes_supported (vectype, group_size, true));
1454 0 : if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
1455 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1456 0 : else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
1457 0 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1458 : scalar_mask);
1459 : else
1460 : {
1461 0 : if (dump_enabled_p ())
1462 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1463 : "can't operate on partial vectors because"
1464 : " the target doesn't have an appropriate"
1465 : " load/store-lanes instruction.\n");
1466 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1467 : }
1468 : return;
1469 : }
1470 :
1471 295794 : if (mat_gather_scatter_p (memory_access_type))
1472 : {
1473 1750 : internal_fn ifn = (is_load
1474 1750 : ? IFN_MASK_GATHER_LOAD
1475 : : IFN_MASK_SCATTER_STORE);
1476 434 : internal_fn len_ifn = (is_load
1477 : ? IFN_MASK_LEN_GATHER_LOAD
1478 : : IFN_MASK_LEN_SCATTER_STORE);
1479 1750 : stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
1480 1750 : tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
1481 1750 : ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
1482 1750 : : ls->strided_offset_vectype);
1483 1750 : tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
1484 1750 : int scale = SLP_TREE_GS_SCALE (slp_node);
1485 :
1486 : /* The following "supported" checks just verify what we established in
1487 : get_load_store_type and don't try different offset types.
1488 : Therefore, off_vectype must be a supported offset type. In case
1489 : we chose a different one use this instead. */
1490 1750 : if (ls->supported_offset_vectype)
1491 0 : off_vectype = ls->supported_offset_vectype;
1492 : /* Same for scale. */
1493 1750 : if (ls->supported_scale)
1494 0 : scale = ls->supported_scale;
1495 :
1496 1750 : if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
1497 : memory_type,
1498 : off_vectype, scale,
1499 : elsvals))
1500 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1501 1750 : else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
1502 : memory_type,
1503 : off_vectype, scale,
1504 : elsvals)
1505 1750 : || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
1506 573 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1507 : scalar_mask);
1508 : else
1509 : {
1510 1177 : if (dump_enabled_p ())
1511 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1512 : "can't operate on partial vectors because"
1513 : " the target doesn't have an appropriate"
1514 : " gather load or scatter store instruction.\n");
1515 1177 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1516 : }
1517 : return;
1518 : }
1519 :
1520 294044 : if (memory_access_type != VMAT_CONTIGUOUS)
1521 : {
1522 : /* Element X of the data must come from iteration i * VF + X of the
1523 : scalar loop. We need more work to support other mappings. */
1524 26517 : if (dump_enabled_p ())
1525 728 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1526 : "can't operate on partial vectors because an"
1527 : " access isn't contiguous.\n");
1528 26517 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1529 26517 : return;
1530 : }
1531 :
1532 267527 : if (!VECTOR_MODE_P (vecmode))
1533 : {
1534 0 : if (dump_enabled_p ())
1535 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1536 : "can't operate on partial vectors when emulating"
1537 : " vector operations.\n");
1538 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1539 0 : return;
1540 : }
1541 :
1542 : /* We might load more scalars than we need for permuting SLP loads.
1543 : We checked in get_load_store_type that the extra elements
1544 : don't leak into a new vector. */
1545 358197 : auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
1546 : {
1547 90670 : unsigned int nvectors;
1548 90670 : if (can_div_away_from_zero_p (size, nunits, &nvectors))
1549 90670 : return nvectors;
1550 : gcc_unreachable ();
1551 : };
1552 :
1553 267527 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1554 267527 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1555 267527 : machine_mode mask_mode;
1556 267527 : machine_mode vmode;
1557 267527 : bool using_partial_vectors_p = false;
1558 267527 : if (get_len_load_store_mode
1559 267527 : (vecmode, is_load, nullptr, elsvals).exists (&vmode))
1560 : {
1561 0 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1562 0 : unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
1563 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
1564 0 : using_partial_vectors_p = true;
1565 : }
1566 358197 : else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
1567 267527 : && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
1568 : elsvals))
1569 : {
1570 90670 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1571 90670 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
1572 90670 : using_partial_vectors_p = true;
1573 : }
1574 :
1575 90670 : if (!using_partial_vectors_p)
1576 : {
1577 176857 : if (dump_enabled_p ())
1578 11923 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1579 : "can't operate on partial vectors because the"
1580 : " target doesn't have the appropriate partial"
1581 : " vectorization load or store.\n");
1582 176857 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1583 : }
1584 : }
1585 :
1586 : /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1587 : form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1588 : that needs to be applied to all loads and stores in a vectorized loop.
1589 : Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1590 : otherwise return VEC_MASK & LOOP_MASK.
1591 :
1592 : MASK_TYPE is the type of both masks. If new statements are needed,
1593 : insert them before GSI. */
1594 :
1595 : tree
1596 1611 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
1597 : tree vec_mask, gimple_stmt_iterator *gsi)
1598 : {
1599 1611 : gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1600 1611 : if (!loop_mask)
1601 : return vec_mask;
1602 :
1603 141 : gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1604 :
1605 141 : if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
1606 : return vec_mask;
1607 :
1608 141 : tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1609 141 : gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1610 : vec_mask, loop_mask);
1611 :
1612 141 : gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1613 141 : return and_res;
1614 : }
1615 :
1616 : /* Determine whether we can use a gather load or scatter store to vectorize
1617 : strided load or store STMT_INFO by truncating the current offset to a
1618 : smaller width. We need to be able to construct an offset vector:
1619 :
1620 : { 0, X, X*2, X*3, ... }
1621 :
1622 : without loss of precision, where X is STMT_INFO's DR_STEP.
1623 :
1624 : Return true if this is possible, describing the gather load or scatter
1625 : store in GS_INFO. MASKED_P is true if the load or store is conditional.
1626 :
1627 : If we can use gather/scatter and ELSVALS is nonzero the supported
1628 : else values will be stored in the vector ELSVALS points to. */
1629 :
1630 : static bool
1631 64567 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
1632 : loop_vec_info loop_vinfo, bool masked_p,
1633 : gather_scatter_info *gs_info,
1634 : vec<int> *elsvals)
1635 : {
1636 64567 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1637 64567 : data_reference *dr = dr_info->dr;
1638 64567 : tree step = DR_STEP (dr);
1639 64567 : if (TREE_CODE (step) != INTEGER_CST)
1640 : {
1641 : /* ??? Perhaps we could use range information here? */
1642 29157 : if (dump_enabled_p ())
1643 203 : dump_printf_loc (MSG_NOTE, vect_location,
1644 : "cannot truncate variable step.\n");
1645 : return false;
1646 : }
1647 :
1648 : /* Get the number of bits in an element. */
1649 35410 : scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1650 35410 : unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1651 :
1652 : /* Set COUNT to the upper limit on the number of elements - 1.
1653 : Start with the maximum vectorization factor. */
1654 35410 : unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1655 :
1656 : /* Try lowering COUNT to the number of scalar latch iterations. */
1657 35410 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1658 35410 : widest_int max_iters;
1659 35410 : if (max_loop_iterations (loop, &max_iters)
1660 70090 : && max_iters < count)
1661 2119 : count = max_iters.to_shwi ();
1662 :
1663 : /* Try scales of 1 and the element size. */
1664 35410 : unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
1665 35410 : wi::overflow_type overflow = wi::OVF_NONE;
1666 106230 : for (int i = 0; i < 2; ++i)
1667 : {
1668 70820 : unsigned int scale = scales[i];
1669 70820 : widest_int factor;
1670 70820 : if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1671 0 : continue;
1672 :
1673 : /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1674 70820 : widest_int range = wi::mul (count, factor, SIGNED, &overflow);
1675 70820 : if (overflow)
1676 0 : continue;
1677 70820 : signop sign = range >= 0 ? UNSIGNED : SIGNED;
1678 70820 : unsigned int min_offset_bits = wi::min_precision (range, sign);
1679 :
1680 : /* Find the narrowest viable offset type. */
1681 70820 : unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
1682 70820 : tree offset_type = build_nonstandard_integer_type (offset_bits,
1683 : sign == UNSIGNED);
1684 :
1685 : /* See whether the target supports the operation with an offset
1686 : no narrower than OFFSET_TYPE. */
1687 70820 : tree memory_type = TREE_TYPE (DR_REF (dr));
1688 70820 : tree tmp_offset_vectype;
1689 70820 : int tmp_scale;
1690 70820 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
1691 : vectype, memory_type, offset_type,
1692 : scale, &tmp_scale,
1693 : &gs_info->ifn, &gs_info->offset_vectype,
1694 : &tmp_offset_vectype, elsvals)
1695 70820 : || gs_info->ifn == IFN_LAST)
1696 70820 : continue;
1697 :
1698 0 : gs_info->decl = NULL_TREE;
1699 : /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1700 : but we don't need to store that here. */
1701 0 : gs_info->base = NULL_TREE;
1702 0 : gs_info->alias_ptr = build_int_cst
1703 0 : (reference_alias_ptr_type (DR_REF (dr)),
1704 0 : get_object_alignment (DR_REF (dr)));
1705 0 : gs_info->element_type = TREE_TYPE (vectype);
1706 0 : gs_info->offset = fold_convert (offset_type, step);
1707 0 : gs_info->scale = scale;
1708 0 : gs_info->memory_type = memory_type;
1709 0 : return true;
1710 70820 : }
1711 :
1712 35410 : if (overflow && dump_enabled_p ())
1713 0 : dump_printf_loc (MSG_NOTE, vect_location,
1714 : "truncating gather/scatter offset to %d bits"
1715 : " might change its value.\n", element_bits);
1716 :
1717 : return false;
1718 64567 : }
1719 :
1720 : /* Return true if we can use gather/scatter or strided internal functions
1721 : to vectorize STMT_INFO, which is a grouped or strided load or store
1722 : with multiple lanes and will be implemented by a type-punned access
1723 : of a vector with element size that matches the number of lanes.
1724 :
1725 : MASKED_P is true if load or store is conditional.
1726 : When returning true, fill in GS_INFO with the information required to
1727 : perform the operation. Also, store the punning type in PUNNED_VECTYPE.
1728 :
1729 : If successful and ELSVALS is nonzero the supported
1730 : else values will be stored in the vector ELSVALS points to. */
1731 :
1732 : static bool
1733 4619 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
1734 : loop_vec_info loop_vinfo, bool masked_p,
1735 : unsigned int nelts,
1736 : gather_scatter_info *info, vec<int> *elsvals,
1737 : tree *pun_vectype)
1738 : {
1739 4619 : data_reference *dr = dr_info->dr;
1740 :
1741 : /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
1742 : multiple gathers/scatter. */
1743 4619 : if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
1744 : return false;
1745 :
1746 : /* Pun the vectype with one of the same size but an element spanning
1747 : NELTS elements of VECTYPE.
1748 : The punned type of a V16QI with NELTS = 4 would be V4SI.
1749 : */
1750 3981 : tree tmp;
1751 3981 : unsigned int pieces;
1752 3981 : if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
1753 3981 : || pieces <= 1)
1754 1993 : return false;
1755 :
1756 1988 : *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
1757 :
1758 1988 : if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
1759 : return false;
1760 :
1761 1842 : internal_fn ifn;
1762 1842 : tree offset_vectype = *pun_vectype;
1763 :
1764 1261 : internal_fn strided_ifn = DR_IS_READ (dr)
1765 1842 : ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
1766 :
1767 : /* Check if we have a gather/scatter with the new type. We're just trying
1768 : with the type itself as offset for now. If not, check if we have a
1769 : strided load/store. These have fewer constraints (for example no offset
1770 : type must exist) so it is possible that even though a gather/scatter is
1771 : not available we still have a strided load/store. */
1772 1842 : bool ok = false;
1773 1842 : tree tmp_vectype;
1774 1842 : int tmp_scale;
1775 1842 : if (vect_gather_scatter_fn_p
1776 1842 : (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
1777 1842 : TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
1778 : &offset_vectype, &tmp_vectype, elsvals))
1779 : ok = true;
1780 1842 : else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
1781 : elsvals))
1782 : {
1783 : /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
1784 : will switch back to the strided variants. */
1785 0 : ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
1786 : IFN_MASK_LEN_SCATTER_STORE;
1787 0 : ok = true;
1788 : }
1789 :
1790 0 : if (ok)
1791 : {
1792 0 : info->ifn = ifn;
1793 0 : info->decl = NULL_TREE;
1794 0 : info->base = dr->ref;
1795 0 : info->alias_ptr = build_int_cst
1796 0 : (reference_alias_ptr_type (DR_REF (dr)),
1797 0 : get_object_alignment (DR_REF (dr)));
1798 0 : info->element_type = TREE_TYPE (*pun_vectype);
1799 0 : info->offset_vectype = offset_vectype;
1800 : /* No need to set the offset, vect_get_strided_load_store_ops
1801 : will do that. */
1802 0 : info->scale = 1;
1803 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
1804 0 : return true;
1805 : }
1806 :
1807 : return false;
1808 : }
1809 :
1810 :
1811 : /* Return true if we can use gather/scatter internal functions to
1812 : vectorize STMT_INFO, which is a grouped or strided load or store.
1813 : MASKED_P is true if load or store is conditional. When returning
1814 : true, fill in GS_INFO with the information required to perform the
1815 : operation.
1816 :
1817 : If we can use gather/scatter and ELSVALS is nonzero the supported
1818 : else values will be stored in the vector ELSVALS points to. */
1819 :
1820 : static bool
1821 64567 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
1822 : loop_vec_info loop_vinfo, bool masked_p,
1823 : gather_scatter_info *gs_info,
1824 : vec<int> *elsvals,
1825 : unsigned int group_size,
1826 : bool single_element_p)
1827 : {
1828 64567 : if (!vect_check_gather_scatter (stmt_info, vectype,
1829 : loop_vinfo, gs_info, elsvals)
1830 64567 : || gs_info->ifn == IFN_LAST)
1831 : {
1832 64567 : if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
1833 : masked_p, gs_info, elsvals))
1834 : return false;
1835 : }
1836 :
1837 0 : if (!single_element_p
1838 0 : && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
1839 : gs_info->scale,
1840 : group_size))
1841 : return false;
1842 :
1843 0 : if (dump_enabled_p ())
1844 0 : dump_printf_loc (MSG_NOTE, vect_location,
1845 : "using gather/scatter for strided/grouped access,"
1846 : " scale = %d\n", gs_info->scale);
1847 :
1848 : return true;
1849 : }
1850 :
1851 : /* STMT_INFO is a non-strided load or store, meaning that it accesses
1852 : elements with a known constant step. Return -1 if that step
1853 : is negative, 0 if it is zero, and 1 if it is greater than zero. */
1854 :
1855 : int
1856 1517237 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
1857 : {
1858 1517237 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1859 1517237 : return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
1860 1517237 : size_zero_node);
1861 : }
1862 :
1863 : /* If the target supports a permute mask that reverses the elements in
1864 : a vector of type VECTYPE, return that mask, otherwise return null. */
1865 :
1866 : tree
1867 9179 : perm_mask_for_reverse (tree vectype)
1868 : {
1869 9179 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1870 :
1871 : /* The encoding has a single stepped pattern. */
1872 9179 : vec_perm_builder sel (nunits, 1, 3);
1873 45895 : for (int i = 0; i < 3; ++i)
1874 27537 : sel.quick_push (nunits - 1 - i);
1875 :
1876 9179 : vec_perm_indices indices (sel, 1, nunits);
1877 9179 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
1878 : indices))
1879 : return NULL_TREE;
1880 8009 : return vect_gen_perm_mask_checked (vectype, indices);
1881 9179 : }
1882 :
1883 : /* A subroutine of get_load_store_type, with a subset of the same
1884 : arguments. Handle the case where STMT_INFO is a load or store that
1885 : accesses consecutive elements with a negative step. Sets *POFFSET
1886 : to the offset to be applied to the DR for the first access. */
1887 :
1888 : static vect_memory_access_type
1889 12308 : get_negative_load_store_type (vec_info *vinfo,
1890 : stmt_vec_info stmt_info, tree vectype,
1891 : vec_load_store_type vls_type,
1892 : unsigned int ncopies, poly_int64 *poffset)
1893 : {
1894 12308 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1895 12308 : dr_alignment_support alignment_support_scheme;
1896 :
1897 12308 : if (ncopies > 1)
1898 : {
1899 0 : if (dump_enabled_p ())
1900 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1901 : "multiple types with negative step.\n");
1902 : return VMAT_ELEMENTWISE;
1903 : }
1904 :
1905 : /* For backward running DRs the first access in vectype actually is
1906 : N-1 elements before the address of the DR. */
1907 12308 : *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
1908 12308 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1909 :
1910 12308 : int misalignment = dr_misalignment (dr_info, vectype, *poffset);
1911 12308 : alignment_support_scheme
1912 12308 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
1913 12308 : if (alignment_support_scheme != dr_aligned
1914 12308 : && alignment_support_scheme != dr_unaligned_supported)
1915 : {
1916 4512 : if (dump_enabled_p ())
1917 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1918 : "negative step but alignment required.\n");
1919 4512 : *poffset = 0;
1920 4512 : return VMAT_ELEMENTWISE;
1921 : }
1922 :
1923 7796 : if (vls_type == VLS_STORE_INVARIANT)
1924 : {
1925 1197 : if (dump_enabled_p ())
1926 21 : dump_printf_loc (MSG_NOTE, vect_location,
1927 : "negative step with invariant source;"
1928 : " no permute needed.\n");
1929 : return VMAT_CONTIGUOUS_DOWN;
1930 : }
1931 :
1932 6599 : if (!perm_mask_for_reverse (vectype))
1933 : {
1934 1170 : if (dump_enabled_p ())
1935 52 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1936 : "negative step and reversing not supported.\n");
1937 1170 : *poffset = 0;
1938 1170 : return VMAT_ELEMENTWISE;
1939 : }
1940 :
1941 : return VMAT_CONTIGUOUS_REVERSE;
1942 : }
1943 :
1944 : /* STMT_INFO is either a masked or unconditional store. Return the value
1945 : being stored. */
1946 :
1947 : tree
1948 0 : vect_get_store_rhs (stmt_vec_info stmt_info)
1949 : {
1950 0 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
1951 : {
1952 0 : gcc_assert (gimple_assign_single_p (assign));
1953 0 : return gimple_assign_rhs1 (assign);
1954 : }
1955 0 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
1956 : {
1957 0 : internal_fn ifn = gimple_call_internal_fn (call);
1958 0 : int index = internal_fn_stored_value_index (ifn);
1959 0 : gcc_assert (index >= 0);
1960 0 : return gimple_call_arg (call, index);
1961 : }
1962 0 : gcc_unreachable ();
1963 : }
1964 :
1965 : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1966 :
1967 : This function returns a vector type which can be composed with NELTS pieces,
1968 : whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1969 : same vector size as the return vector. It checks target whether supports
1970 : pieces-size vector mode for construction firstly, if target fails to, check
1971 : pieces-size scalar mode for construction further. It returns NULL_TREE if
1972 : fails to find the available composition. If the caller only wants scalar
1973 : pieces where PTYPE e.g. is a possible gather/scatter element type
1974 : SCALAR_PTYPE_ONLY must be true.
1975 :
1976 : For example, for (vtype=V16QI, nelts=4), we can probably get:
1977 : - V16QI with PTYPE V4QI.
1978 : - V4SI with PTYPE SI.
1979 : - NULL_TREE. */
1980 :
1981 : static tree
1982 12676 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
1983 : bool scalar_ptype_only)
1984 : {
1985 12676 : gcc_assert (VECTOR_TYPE_P (vtype));
1986 12676 : gcc_assert (known_gt (nelts, 0U));
1987 :
1988 12676 : machine_mode vmode = TYPE_MODE (vtype);
1989 12676 : if (!VECTOR_MODE_P (vmode))
1990 : return NULL_TREE;
1991 :
1992 : /* When we are asked to compose the vector from its components let
1993 : that happen directly. */
1994 12676 : if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
1995 : {
1996 6093 : *ptype = TREE_TYPE (vtype);
1997 6093 : return vtype;
1998 : }
1999 :
2000 13166 : poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
2001 6583 : unsigned int pbsize;
2002 6583 : if (constant_multiple_p (vbsize, nelts, &pbsize))
2003 : {
2004 : /* First check if vec_init optab supports construction from
2005 : vector pieces directly. */
2006 6583 : scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
2007 13166 : poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
2008 6583 : machine_mode rmode;
2009 6583 : if (!scalar_ptype_only
2010 8088 : && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
2011 10716 : && (convert_optab_handler (vec_init_optab, vmode, rmode)
2012 : != CODE_FOR_nothing))
2013 : {
2014 3493 : *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
2015 3493 : return vtype;
2016 : }
2017 :
2018 : /* Otherwise check if exists an integer type of the same piece size and
2019 : if vec_init optab supports construction from it directly. */
2020 3090 : if (int_mode_for_size (pbsize, 0).exists (&elmode)
2021 3090 : && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
2022 : {
2023 2906 : if (scalar_ptype_only
2024 2906 : || convert_optab_handler (vec_init_optab, rmode, elmode)
2025 : != CODE_FOR_nothing)
2026 : {
2027 2906 : *ptype = build_nonstandard_integer_type (pbsize, 1);
2028 2906 : return build_vector_type (*ptype, nelts);
2029 : }
2030 : }
2031 : }
2032 :
2033 : return NULL_TREE;
2034 : }
2035 :
2036 : /* Check if the load permutation of NODE only refers to a consecutive
2037 : subset of the group indices where GROUP_SIZE is the size of the
2038 : dataref's group. We also assert that the length of the permutation
2039 : divides the group size and is a power of two.
2040 : Such load permutations can be elided in strided access schemes as
2041 : we can "jump over" the gap they leave. */
2042 :
2043 : bool
2044 45627 : has_consecutive_load_permutation (slp_tree node, unsigned group_size)
2045 : {
2046 45627 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
2047 45627 : if (!perm.exists ()
2048 2164 : || perm.length () <= 1
2049 496 : || !pow2p_hwi (perm.length ())
2050 46107 : || group_size % perm.length ())
2051 : return false;
2052 :
2053 433 : return vect_load_perm_consecutive_p (node);
2054 : }
2055 :
2056 :
2057 : /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
2058 : if there is a memory access type that the vectorized form can use,
2059 : storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2060 : or scatters, fill in GS_INFO accordingly. In addition
2061 : *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2062 : the target does not support the alignment scheme. *MISALIGNMENT
2063 : is set according to the alignment of the access (including
2064 : DR_MISALIGNMENT_UNKNOWN when it is unknown).
2065 :
2066 : MASKED_P is true if the statement is conditional on a vectorized mask.
2067 : VECTYPE is the vector type that the vectorized statements will use.
2068 :
2069 : If ELSVALS is nonzero the supported else values will be stored in the
2070 : vector ELSVALS points to. */
2071 :
2072 : static bool
2073 1402575 : get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
2074 : tree vectype, slp_tree slp_node,
2075 : bool masked_p, vec_load_store_type vls_type,
2076 : vect_load_store_data *ls)
2077 : {
2078 1402575 : vect_memory_access_type *memory_access_type = &ls->memory_access_type;
2079 1402575 : poly_int64 *poffset = &ls->poffset;
2080 1402575 : dr_alignment_support *alignment_support_scheme
2081 : = &ls->alignment_support_scheme;
2082 1402575 : int *misalignment = &ls->misalignment;
2083 1402575 : internal_fn *lanes_ifn = &ls->lanes_ifn;
2084 1402575 : vec<int> *elsvals = &ls->elsvals;
2085 1402575 : tree *ls_type = &ls->ls_type;
2086 1402575 : bool *slp_perm = &ls->slp_perm;
2087 1402575 : unsigned *n_perms = &ls->n_perms;
2088 1402575 : unsigned *n_loads = &ls->n_loads;
2089 1402575 : tree *supported_offset_vectype = &ls->supported_offset_vectype;
2090 1402575 : int *supported_scale = &ls->supported_scale;
2091 1402575 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
2092 1402575 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2093 1402575 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2094 1402575 : stmt_vec_info first_stmt_info;
2095 1402575 : unsigned int group_size;
2096 1402575 : unsigned HOST_WIDE_INT gap;
2097 1402575 : bool single_element_p;
2098 1402575 : poly_int64 neg_ldst_offset = 0;
2099 :
2100 1402575 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2101 1402575 : *poffset = 0;
2102 1402575 : *ls_type = NULL_TREE;
2103 1402575 : *slp_perm = false;
2104 1402575 : *n_perms = -1U;
2105 1402575 : *n_loads = -1U;
2106 1402575 : ls->subchain_p = false;
2107 :
2108 1402575 : bool perm_ok = true;
2109 1402575 : poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
2110 :
2111 1402575 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2112 80516 : perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
2113 80516 : vf, true, n_perms, n_loads);
2114 :
2115 1402575 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2116 : {
2117 898077 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
2118 898077 : group_size = DR_GROUP_SIZE (first_stmt_info);
2119 898077 : gap = DR_GROUP_GAP (first_stmt_info);
2120 898077 : single_element_p = (stmt_info == first_stmt_info
2121 898077 : && !DR_GROUP_NEXT_ELEMENT (stmt_info));
2122 : }
2123 : else
2124 : {
2125 : first_stmt_info = stmt_info;
2126 : group_size = 1;
2127 : gap = 0;
2128 : single_element_p = true;
2129 : }
2130 1402575 : dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
2131 :
2132 : /* True if the vectorized statements would access beyond the last
2133 : statement in the group. */
2134 1402575 : bool overrun_p = false;
2135 :
2136 : /* True if we can cope with such overrun by peeling for gaps, so that
2137 : there is at least one final scalar iteration after the vector loop. */
2138 2805150 : bool can_overrun_p = (!masked_p
2139 1402575 : && vls_type == VLS_LOAD
2140 568703 : && loop_vinfo
2141 1830399 : && !loop->inner);
2142 :
2143 : /* There can only be a gap at the end of the group if the stride is
2144 : known at compile time. */
2145 1402575 : gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
2146 :
2147 : /* For SLP vectorization we directly vectorize a subchain
2148 : without permutation. */
2149 1402575 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2150 1322059 : first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
2151 :
2152 1402575 : if (STMT_VINFO_STRIDED_P (first_stmt_info))
2153 : {
2154 : /* Try to use consecutive accesses of as many elements as possible,
2155 : separated by the stride, until we have a complete vector.
2156 : Fall back to scalar accesses if that isn't possible. */
2157 45627 : *memory_access_type = VMAT_STRIDED_SLP;
2158 :
2159 : /* If the load permutation is consecutive we can reduce the group to
2160 : the elements the permutation accesses. Then we release the
2161 : permutation. */
2162 45627 : if (has_consecutive_load_permutation (slp_node, group_size))
2163 : {
2164 32 : ls->subchain_p = true;
2165 32 : group_size = SLP_TREE_LANES (slp_node);
2166 32 : SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
2167 : }
2168 : }
2169 1356948 : else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2170 : {
2171 10944 : slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
2172 10944 : tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
2173 10944 : int scale = SLP_TREE_GS_SCALE (slp_node);
2174 10944 : tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
2175 10944 : tree tem;
2176 10944 : if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
2177 : masked_p, vectype, memory_type,
2178 : offset_vectype, scale, supported_scale,
2179 : &ls->gs.ifn, &tem,
2180 : supported_offset_vectype, elsvals))
2181 : {
2182 0 : if (dump_enabled_p ())
2183 : {
2184 0 : dump_printf_loc (MSG_NOTE, vect_location,
2185 : "gather/scatter with required "
2186 : "offset type "
2187 : "%T and offset scale %d.\n",
2188 : offset_vectype, scale);
2189 0 : if (*supported_offset_vectype)
2190 0 : dump_printf_loc (MSG_NOTE, vect_location,
2191 : " target supports offset type %T.\n",
2192 : *supported_offset_vectype);
2193 0 : if (*supported_scale)
2194 0 : dump_printf_loc (MSG_NOTE, vect_location,
2195 : " target supports offset scale %d.\n",
2196 : *supported_scale);
2197 : }
2198 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2199 : }
2200 10944 : else if (vls_type == VLS_LOAD
2201 10944 : ? (targetm.vectorize.builtin_gather
2202 9337 : && (ls->gs.decl
2203 9337 : = targetm.vectorize.builtin_gather (vectype,
2204 9337 : TREE_TYPE
2205 : (offset_vectype),
2206 : scale)))
2207 1607 : : (targetm.vectorize.builtin_scatter
2208 1607 : && (ls->gs.decl
2209 1607 : = targetm.vectorize.builtin_scatter (vectype,
2210 1607 : TREE_TYPE
2211 : (offset_vectype),
2212 : scale))))
2213 581 : *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
2214 : else
2215 : {
2216 : /* GATHER_SCATTER_EMULATED_P. */
2217 10363 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
2218 10363 : || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
2219 10363 : || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
2220 10363 : || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
2221 13107 : TYPE_VECTOR_SUBPARTS (vectype)))
2222 : {
2223 2744 : if (dump_enabled_p ())
2224 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2225 : "unsupported vector types for emulated "
2226 : "gather.\n");
2227 2744 : return false;
2228 : }
2229 7619 : *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
2230 : }
2231 : }
2232 : else
2233 : {
2234 1346004 : int cmp = compare_step_with_zero (vinfo, stmt_info);
2235 1346004 : if (cmp < 0)
2236 : {
2237 12486 : if (single_element_p)
2238 : /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2239 : only correct for single element "interleaving" SLP. */
2240 12308 : *memory_access_type = get_negative_load_store_type
2241 12308 : (vinfo, stmt_info, vectype, vls_type, 1,
2242 : &neg_ldst_offset);
2243 : else
2244 : /* We can fall back to VMAT_STRIDED_SLP since that does
2245 : not care whether the stride between the group instances
2246 : is positive or negative. */
2247 178 : *memory_access_type = VMAT_STRIDED_SLP;
2248 : }
2249 1333518 : else if (cmp == 0 && loop_vinfo)
2250 : {
2251 3386 : gcc_assert (vls_type == VLS_LOAD);
2252 3386 : *memory_access_type = VMAT_INVARIANT;
2253 : }
2254 : /* Try using LOAD/STORE_LANES. */
2255 1330132 : else if (slp_node->ldst_lanes
2256 1330132 : && (*lanes_ifn
2257 0 : = (vls_type == VLS_LOAD
2258 0 : ? vect_load_lanes_supported (vectype, group_size,
2259 : masked_p, elsvals)
2260 0 : : vect_store_lanes_supported (vectype, group_size,
2261 : masked_p))) != IFN_LAST)
2262 0 : *memory_access_type = VMAT_LOAD_STORE_LANES;
2263 1330132 : else if (!loop_vinfo && slp_node->avoid_stlf_fail)
2264 : {
2265 70 : *memory_access_type = VMAT_ELEMENTWISE;
2266 70 : if (dump_enabled_p ())
2267 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2268 : "using element-wise load to avoid disrupting "
2269 : "cross iteration store-to-load forwarding\n");
2270 : }
2271 : else
2272 1330062 : *memory_access_type = VMAT_CONTIGUOUS;
2273 :
2274 : /* If this is single-element interleaving with an element
2275 : distance that leaves unused vector loads around fall back
2276 : to elementwise access if possible - we otherwise least
2277 : create very sub-optimal code in that case (and
2278 : blow up memory, see PR65518). */
2279 1346004 : if (loop_vinfo
2280 1346004 : && single_element_p
2281 484592 : && (*memory_access_type == VMAT_CONTIGUOUS
2282 15694 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2283 1830596 : && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
2284 : {
2285 17845 : *memory_access_type = VMAT_ELEMENTWISE;
2286 17845 : if (dump_enabled_p ())
2287 198 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2288 : "single-element interleaving not supported "
2289 : "for not adjacent vector loads, using "
2290 : "elementwise access\n");
2291 : }
2292 :
2293 : /* Also fall back to elementwise access in case we did not lower a
2294 : permutation and cannot code generate it. */
2295 1346004 : if (loop_vinfo
2296 539133 : && *memory_access_type != VMAT_ELEMENTWISE
2297 515606 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2298 1374666 : && !perm_ok)
2299 : {
2300 2081 : *memory_access_type = VMAT_ELEMENTWISE;
2301 2081 : if (dump_enabled_p ())
2302 248 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2303 : "permutation not supported, using elementwise "
2304 : "access\n");
2305 : }
2306 :
2307 539133 : overrun_p = (loop_vinfo && gap != 0
2308 1389057 : && *memory_access_type != VMAT_ELEMENTWISE);
2309 1346004 : if (overrun_p && vls_type != VLS_LOAD)
2310 : {
2311 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2312 : "Grouped store with gaps requires"
2313 : " non-consecutive accesses\n");
2314 0 : return false;
2315 : }
2316 :
2317 1346004 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
2318 1346004 : poly_int64 off = 0;
2319 1346004 : if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2320 5270 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2321 :
2322 : /* An overrun is fine if the trailing elements are smaller
2323 : than the alignment boundary B. Every vector access will
2324 : be a multiple of B and so we are guaranteed to access a
2325 : non-gap element in the same B-sized block. */
2326 1346004 : if (overrun_p
2327 1346004 : && gap < (vect_known_alignment_in_bytes (first_dr_info,
2328 23042 : vectype, off) / dr_size))
2329 : overrun_p = false;
2330 :
2331 : /* When we have a contiguous access across loop iterations
2332 : but the access in the loop doesn't cover the full vector
2333 : we can end up with no gap recorded but still excess
2334 : elements accessed, see PR103116. Make sure we peel for
2335 : gaps if necessary and sufficient and give up if not.
2336 :
2337 : If there is a combination of the access not covering the full
2338 : vector and a gap recorded then we may need to peel twice. */
2339 1346004 : bool large_vector_overrun_p = false;
2340 1346004 : if (loop_vinfo
2341 539133 : && (*memory_access_type == VMAT_CONTIGUOUS
2342 35634 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2343 508769 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2344 1372206 : && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
2345 : nunits))
2346 : large_vector_overrun_p = overrun_p = true;
2347 :
2348 : /* If the gap splits the vector in half and the target
2349 : can do half-vector operations avoid the epilogue peeling
2350 : by simply loading half of the vector only. Usually
2351 : the construction with an upper zero half will be elided. */
2352 1346004 : dr_alignment_support alss;
2353 1346004 : int misalign = dr_misalignment (first_dr_info, vectype, off);
2354 1346004 : tree half_vtype;
2355 1346004 : poly_uint64 remain;
2356 1346004 : unsigned HOST_WIDE_INT tem, num;
2357 1346004 : if (overrun_p
2358 1346004 : && !masked_p
2359 17554 : && *memory_access_type != VMAT_LOAD_STORE_LANES
2360 17554 : && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
2361 : vectype, misalign)))
2362 : == dr_aligned
2363 15054 : || alss == dr_unaligned_supported)
2364 9915 : && can_div_trunc_p (group_size
2365 9915 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2366 : nunits, &tem, &remain)
2367 1355919 : && (known_eq (remain, 0u)
2368 7436 : || (known_ne (remain, 0u)
2369 5758 : && constant_multiple_p (nunits, remain, &num)
2370 1346004 : && (vector_vector_composition_type (vectype, num, &half_vtype)
2371 : != NULL_TREE))))
2372 : overrun_p = false;
2373 :
2374 1346004 : if (overrun_p && !can_overrun_p)
2375 : {
2376 6 : if (dump_enabled_p ())
2377 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2378 : "Peeling for outer loop is not supported\n");
2379 : return false;
2380 : }
2381 :
2382 : /* Peeling for gaps assumes that a single scalar iteration
2383 : is enough to make sure the last vector iteration doesn't
2384 : access excess elements. */
2385 1345998 : if (overrun_p
2386 1345998 : && (!can_div_trunc_p (group_size
2387 9311 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2388 : nunits, &tem, &remain)
2389 9311 : || maybe_lt (remain + group_size, nunits)))
2390 : {
2391 : /* But peeling a single scalar iteration is enough if
2392 : we can use the next power-of-two sized partial
2393 : access and that is sufficiently small to be covered
2394 : by the single scalar iteration. */
2395 16 : unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
2396 16 : if (masked_p
2397 16 : || !nunits.is_constant (&cnunits)
2398 16 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
2399 16 : || (((cremain = (group_size * cvf - gap) % cnunits), true)
2400 16 : && ((cpart_size = (1 << ceil_log2 (cremain))), true)
2401 16 : && (cremain + group_size < cpart_size
2402 16 : || (vector_vector_composition_type (vectype,
2403 13 : cnunits / cpart_size,
2404 : &half_vtype)
2405 : == NULL_TREE))))
2406 : {
2407 : /* If all fails we can still resort to niter masking unless
2408 : the vectors used are too big, so enforce the use of
2409 : partial vectors. */
2410 3 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2411 3 : && !large_vector_overrun_p)
2412 : {
2413 0 : if (dump_enabled_p ())
2414 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2415 : "peeling for gaps insufficient for "
2416 : "access unless using partial "
2417 : "vectors\n");
2418 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2419 : }
2420 : else
2421 : {
2422 3 : if (dump_enabled_p ())
2423 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2424 : "peeling for gaps insufficient for "
2425 : "access\n");
2426 9 : return false;
2427 : }
2428 : }
2429 13 : else if (large_vector_overrun_p)
2430 : {
2431 13 : if (dump_enabled_p ())
2432 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2433 : "can't operate on partial vectors because "
2434 : "only unmasked loads handle access "
2435 : "shortening required because of gaps at "
2436 : "the end of the access\n");
2437 13 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2438 : }
2439 : }
2440 : }
2441 :
2442 : /* As a last resort, trying using a gather load or scatter store.
2443 :
2444 : ??? Although the code can handle all group sizes correctly,
2445 : it probably isn't a win to use separate strided accesses based
2446 : on nearby locations. Or, even if it's a win over scalar code,
2447 : it might not be a win over vectorizing at a lower VF, if that
2448 : allows us to use contiguous accesses. */
2449 1399822 : vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
2450 1399822 : if (loop_vinfo
2451 592951 : && (*memory_access_type == VMAT_ELEMENTWISE
2452 592951 : || *memory_access_type == VMAT_STRIDED_SLP))
2453 : {
2454 71408 : gather_scatter_info gs_info;
2455 71408 : tree tem;
2456 71408 : if (SLP_TREE_LANES (slp_node) == 1
2457 66595 : && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2458 21644 : || single_element_p)
2459 135975 : && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
2460 : masked_p, &gs_info, elsvals,
2461 : group_size, single_element_p))
2462 : {
2463 : /* vect_use_strided_gather_scatters_p does not save the actually
2464 : supported scale and offset type so do that here.
2465 : We need it later in check_load_store_for_partial_vectors
2466 : where we only check if the given internal function is supported
2467 : (to choose whether to use the IFN, LEGACY, or EMULATED flavor
2468 : of gather/scatter) and don't re-do the full analysis. */
2469 0 : tree tmp;
2470 0 : gcc_assert (vect_gather_scatter_fn_p
2471 : (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
2472 : gs_info.memory_type, TREE_TYPE (gs_info.offset),
2473 : gs_info.scale, supported_scale, &gs_info.ifn,
2474 : &tmp, supported_offset_vectype, elsvals));
2475 :
2476 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2477 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2478 0 : ls->gs.ifn = gs_info.ifn;
2479 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2480 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2481 : }
2482 71408 : else if (SLP_TREE_LANES (slp_node) > 1
2483 : && !masked_p
2484 4813 : && !single_element_p
2485 76027 : && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
2486 : vectype, loop_vinfo,
2487 : masked_p, group_size,
2488 : &gs_info, elsvals, &tem))
2489 : {
2490 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2491 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2492 0 : grouped_gather_fallback = *memory_access_type;
2493 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2494 0 : ls->gs.ifn = gs_info.ifn;
2495 0 : vectype = *ls_type = tem;
2496 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2497 : }
2498 : }
2499 :
2500 1399822 : if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
2501 1399822 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2502 6463 : *poffset = neg_ldst_offset;
2503 :
2504 1399822 : if (*memory_access_type == VMAT_ELEMENTWISE
2505 1374144 : || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
2506 1373563 : || *memory_access_type == VMAT_STRIDED_SLP
2507 1327763 : || *memory_access_type == VMAT_INVARIANT)
2508 : {
2509 75445 : *alignment_support_scheme = dr_unaligned_supported;
2510 75445 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2511 : }
2512 : else
2513 : {
2514 1324377 : if (mat_gather_scatter_p (*memory_access_type)
2515 : && !first_dr_info)
2516 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2517 : else
2518 1324377 : *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
2519 1324377 : *alignment_support_scheme
2520 1324377 : = vect_supportable_dr_alignment
2521 1324377 : (vinfo, first_dr_info, vectype, *misalignment,
2522 : mat_gather_scatter_p (*memory_access_type));
2523 1324377 : if (grouped_gather_fallback != VMAT_UNINITIALIZED
2524 0 : && *alignment_support_scheme != dr_aligned
2525 0 : && *alignment_support_scheme != dr_unaligned_supported)
2526 : {
2527 : /* No supportable alignment for a grouped gather, fall back to the
2528 : original memory access type. Even though VMAT_STRIDED_SLP might
2529 : also try aligned vector loads it can still choose vector
2530 : construction from scalars. */
2531 0 : *memory_access_type = grouped_gather_fallback;
2532 0 : *alignment_support_scheme = dr_unaligned_supported;
2533 0 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2534 : }
2535 : }
2536 :
2537 1399822 : if (overrun_p)
2538 : {
2539 9308 : gcc_assert (can_overrun_p);
2540 9308 : if (dump_enabled_p ())
2541 511 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2542 : "Data access with gaps requires scalar "
2543 : "epilogue loop\n");
2544 9308 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2545 : }
2546 :
2547 1399822 : if ((*memory_access_type == VMAT_ELEMENTWISE
2548 1399822 : || *memory_access_type == VMAT_STRIDED_SLP)
2549 : && !nunits.is_constant ())
2550 : {
2551 : if (dump_enabled_p ())
2552 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2553 : "Not using elementwise accesses due to variable "
2554 : "vectorization factor.\n");
2555 : return false;
2556 : }
2557 :
2558 : /* Checks if all scalar iterations are known to be inbounds. */
2559 1399822 : bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
2560 :
2561 : /* Check if we support the operation if early breaks are needed. Here we
2562 : must ensure that we don't access any more than the scalar code would
2563 : have. A masked operation would ensure this, so for these load types
2564 : force masking. */
2565 1399822 : if (loop_vinfo
2566 592951 : && dr_safe_speculative_read_required (stmt_info)
2567 1592381 : && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2568 : {
2569 192559 : if (mat_gather_scatter_p (*memory_access_type)
2570 192559 : || *memory_access_type == VMAT_STRIDED_SLP)
2571 : {
2572 9480 : if (dump_enabled_p ())
2573 8 : dump_printf_loc (MSG_NOTE, vect_location,
2574 : "early break not supported: cannot peel for "
2575 : "alignment. With non-contiguous memory vectorization"
2576 : " could read out of bounds at %G ",
2577 : STMT_VINFO_STMT (stmt_info));
2578 9480 : if (inbounds)
2579 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2580 : else
2581 : return false;
2582 : }
2583 : /* Block-level alignment: Even though individual accesses of
2584 : VMAT_ELEMENTWISE type do not cause alignment problems, loading the
2585 : whole vector's worth of values in a speculative early-break context
2586 : might cross a page boundary. Set the alignment scheme to `dr_aligned'
2587 : here in order to force checking of whether such accesses meet
2588 : alignment criteria. */
2589 183079 : else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
2590 15026 : *alignment_support_scheme = dr_aligned;
2591 : }
2592 :
2593 : /* If this DR needs alignment for correctness, we must ensure the target
2594 : alignment is a constant power-of-two multiple of the amount read per
2595 : vector iteration or force masking. */
2596 1390342 : if (dr_safe_speculative_read_required (stmt_info)
2597 1390342 : && (*alignment_support_scheme == dr_aligned
2598 110190 : && !mat_gather_scatter_p (*memory_access_type)))
2599 : {
2600 : /* We can only peel for loops, of course. */
2601 110190 : gcc_checking_assert (loop_vinfo);
2602 :
2603 110190 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2604 110190 : poly_uint64 read_amount
2605 110190 : = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
2606 110190 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2607 110190 : read_amount *= group_size;
2608 :
2609 110190 : auto target_alignment
2610 110190 : = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
2611 110190 : if (!multiple_p (target_alignment, read_amount))
2612 : {
2613 12704 : if (dump_enabled_p ())
2614 : {
2615 28 : dump_printf_loc (MSG_NOTE, vect_location,
2616 : "desired alignment not met, target was ");
2617 28 : dump_dec (MSG_NOTE, target_alignment);
2618 28 : dump_printf (MSG_NOTE, " previously, but read amount is ");
2619 28 : dump_dec (MSG_NOTE, read_amount);
2620 28 : dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
2621 : }
2622 14973 : return false;
2623 : }
2624 :
2625 : /* When using a group access the first element may be aligned but the
2626 : subsequent loads may not be. For LOAD_LANES since the loads are based
2627 : on the first DR then all loads in the group are aligned. For
2628 : non-LOAD_LANES this is not the case. In particular a load + blend when
2629 : there are gaps can have the non first loads issued unaligned, even
2630 : partially overlapping the memory of the first load in order to simplify
2631 : the blend. This is what the x86_64 backend does for instance. As
2632 : such only the first load in the group is aligned, the rest are not.
2633 : Because of this the permutes may break the alignment requirements that
2634 : have been set, and as such we should for now, reject them. */
2635 97486 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2636 : {
2637 2269 : if (dump_enabled_p ())
2638 75 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2639 : "loads with load permutations not supported for "
2640 : "speculative early break loads for %G",
2641 : STMT_VINFO_STMT (stmt_info));
2642 : return false;
2643 : }
2644 :
2645 : /* Reject vectorization if we know the read mount per vector iteration
2646 : exceeds the min page size. */
2647 95217 : if (known_gt (read_amount, (unsigned) param_min_pagesize))
2648 : {
2649 0 : if (dump_enabled_p ())
2650 : {
2651 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2652 : "alignment required for correctness (");
2653 0 : dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
2654 0 : dump_printf (MSG_NOTE, ") may exceed page size.\n");
2655 : }
2656 : return false;
2657 : }
2658 :
2659 95217 : if (!vf.is_constant ())
2660 : {
2661 : /* For VLA modes, we need a runtime check to ensure any speculative
2662 : read amount does not exceed the page size. Here we record the max
2663 : possible read amount for the check. */
2664 : if (maybe_gt (read_amount,
2665 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
2666 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
2667 :
2668 : /* For VLA modes, we must use partial vectors. */
2669 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2670 : }
2671 : }
2672 :
2673 1375369 : if (*alignment_support_scheme == dr_unaligned_unsupported)
2674 : {
2675 69502 : if (dump_enabled_p ())
2676 256 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2677 : "unsupported unaligned access\n");
2678 : return false;
2679 : }
2680 :
2681 : /* FIXME: At the moment the cost model seems to underestimate the
2682 : cost of using elementwise accesses. This check preserves the
2683 : traditional behavior until that can be fixed. */
2684 1305867 : if (*memory_access_type == VMAT_ELEMENTWISE
2685 14929 : && !STMT_VINFO_STRIDED_P (first_stmt_info)
2686 1320796 : && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
2687 9686 : && single_element_p
2688 9041 : && !pow2p_hwi (group_size)))
2689 : {
2690 9228 : if (dump_enabled_p ())
2691 364 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2692 : "not falling back to elementwise accesses\n");
2693 : return false;
2694 : }
2695 :
2696 : /* For BB vectorization build up the vector from existing scalar defs. */
2697 1296639 : if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
2698 : return false;
2699 :
2700 : /* Some loads need to explicitly permute the loaded data if there
2701 : is a load permutation. Among those are:
2702 : - VMAT_ELEMENTWISE.
2703 : - VMAT_STRIDED_SLP.
2704 : - VMAT_GATHER_SCATTER:
2705 : - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
2706 : - Grouped strided gather (ditto but for #lanes > 1).
2707 :
2708 : For VMAT_ELEMENTWISE we can fold the load permutation into the
2709 : individual indices we access directly, eliding the permutation.
2710 : Strided gather only allows load permutations for the
2711 : single-element case. */
2712 :
2713 1296639 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2714 1296639 : && !(*memory_access_type == VMAT_ELEMENTWISE
2715 53446 : || (mat_gather_scatter_p (*memory_access_type)
2716 0 : && SLP_TREE_LANES (slp_node) == 1
2717 0 : && single_element_p)))
2718 : {
2719 53446 : if (!loop_vinfo)
2720 : {
2721 : /* In BB vectorization we may not actually use a loaded vector
2722 : accessing elements in excess of DR_GROUP_SIZE. */
2723 31522 : stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
2724 31522 : group_info = DR_GROUP_FIRST_ELEMENT (group_info);
2725 31522 : unsigned HOST_WIDE_INT nunits;
2726 31522 : unsigned j, k, maxk = 0;
2727 110364 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
2728 78842 : if (k > maxk)
2729 : maxk = k;
2730 31522 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2731 57487 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
2732 31522 : || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
2733 : {
2734 5557 : if (dump_enabled_p ())
2735 39 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2736 : "BB vectorization with gaps at the end of "
2737 : "a load is not supported\n");
2738 1402575 : return false;
2739 : }
2740 : }
2741 :
2742 47889 : if (!perm_ok)
2743 : {
2744 2523 : if (dump_enabled_p ())
2745 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
2746 : vect_location,
2747 : "unsupported load permutation\n");
2748 : return false;
2749 : }
2750 :
2751 45366 : *slp_perm = true;
2752 : }
2753 :
2754 : return true;
2755 : }
2756 :
2757 : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2758 : conditional operation STMT_INFO. When returning true, store the mask
2759 : in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
2760 : the vectorized mask in *MASK_VECTYPE_OUT. */
2761 :
2762 : static bool
2763 12641 : vect_check_scalar_mask (vec_info *vinfo,
2764 : slp_tree slp_node, unsigned mask_index,
2765 : slp_tree *mask_node,
2766 : vect_def_type *mask_dt_out, tree *mask_vectype_out)
2767 : {
2768 12641 : enum vect_def_type mask_dt;
2769 12641 : tree mask_vectype;
2770 12641 : slp_tree mask_node_1;
2771 12641 : tree mask_;
2772 12641 : if (!vect_is_simple_use (vinfo, slp_node, mask_index,
2773 : &mask_, &mask_node_1, &mask_dt, &mask_vectype))
2774 : {
2775 0 : if (dump_enabled_p ())
2776 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2777 : "mask use not simple.\n");
2778 : return false;
2779 : }
2780 :
2781 12641 : if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
2782 12641 : && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
2783 : {
2784 0 : if (dump_enabled_p ())
2785 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2786 : "mask argument is not a boolean.\n");
2787 : return false;
2788 : }
2789 :
2790 12641 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2791 12641 : if (!mask_vectype)
2792 19 : mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
2793 : mask_node_1);
2794 :
2795 12641 : if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2796 : {
2797 0 : if (dump_enabled_p ())
2798 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2799 : "could not find an appropriate vector mask type.\n");
2800 : return false;
2801 : }
2802 :
2803 12641 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2804 25282 : TYPE_VECTOR_SUBPARTS (vectype)))
2805 : {
2806 0 : if (dump_enabled_p ())
2807 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2808 : "vector mask type %T"
2809 : " does not match vector data type %T.\n",
2810 : mask_vectype, vectype);
2811 :
2812 : return false;
2813 : }
2814 :
2815 12641 : *mask_dt_out = mask_dt;
2816 12641 : *mask_vectype_out = mask_vectype;
2817 12641 : *mask_node = mask_node_1;
2818 12641 : return true;
2819 : }
2820 :
2821 :
2822 : /* Return true if stored value is suitable for vectorizing store
2823 : statement STMT_INFO. When returning true, store the scalar stored
2824 : in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2825 : the type of the vectorized store value in
2826 : *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2827 :
2828 : static bool
2829 1387430 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
2830 : slp_tree slp_node, slp_tree *rhs_node,
2831 : vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
2832 : vec_load_store_type *vls_type_out)
2833 : {
2834 1387430 : int op_no = 0;
2835 1387430 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
2836 : {
2837 1857 : if (gimple_call_internal_p (call)
2838 1857 : && internal_store_fn_p (gimple_call_internal_fn (call)))
2839 1857 : op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
2840 : }
2841 1387430 : op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
2842 :
2843 1387430 : enum vect_def_type rhs_dt;
2844 1387430 : tree rhs_vectype;
2845 1387430 : tree rhs;
2846 1387430 : if (!vect_is_simple_use (vinfo, slp_node, op_no,
2847 : &rhs, rhs_node, &rhs_dt, &rhs_vectype))
2848 : {
2849 0 : if (dump_enabled_p ())
2850 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2851 : "use not simple.\n");
2852 : return false;
2853 : }
2854 :
2855 : /* In the case this is a store from a constant make sure
2856 : native_encode_expr can handle it. */
2857 1387430 : if (rhs_dt == vect_constant_def
2858 1387430 : && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2859 : {
2860 0 : if (dump_enabled_p ())
2861 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2862 : "cannot encode constant as a byte sequence.\n");
2863 : return false;
2864 : }
2865 :
2866 1387430 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2867 1387430 : if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2868 : {
2869 24 : if (dump_enabled_p ())
2870 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2871 : "incompatible vector types.\n");
2872 : return false;
2873 : }
2874 :
2875 1387406 : *rhs_dt_out = rhs_dt;
2876 1387406 : *rhs_vectype_out = rhs_vectype;
2877 1387406 : if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2878 1024090 : *vls_type_out = VLS_STORE_INVARIANT;
2879 : else
2880 363316 : *vls_type_out = VLS_STORE;
2881 : return true;
2882 : }
2883 :
2884 : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2885 : Note that we support masks with floating-point type, in which case the
2886 : floats are interpreted as a bitmask. */
2887 :
2888 : static tree
2889 170 : vect_build_all_ones_mask (vec_info *vinfo,
2890 : stmt_vec_info stmt_info, tree masktype)
2891 : {
2892 170 : if (TREE_CODE (masktype) == INTEGER_TYPE)
2893 98 : return build_int_cst (masktype, -1);
2894 72 : else if (VECTOR_BOOLEAN_TYPE_P (masktype)
2895 144 : || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2896 : {
2897 19 : tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2898 19 : mask = build_vector_from_val (masktype, mask);
2899 19 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2900 : }
2901 53 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2902 : {
2903 : REAL_VALUE_TYPE r;
2904 : long tmp[6];
2905 371 : for (int j = 0; j < 6; ++j)
2906 318 : tmp[j] = -1;
2907 53 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2908 53 : tree mask = build_real (TREE_TYPE (masktype), r);
2909 53 : mask = build_vector_from_val (masktype, mask);
2910 53 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2911 : }
2912 0 : gcc_unreachable ();
2913 : }
2914 :
2915 : /* Build an all-zero merge value of type VECTYPE while vectorizing
2916 : STMT_INFO as a gather load. */
2917 :
2918 : static tree
2919 158 : vect_build_zero_merge_argument (vec_info *vinfo,
2920 : stmt_vec_info stmt_info, tree vectype)
2921 : {
2922 158 : tree merge;
2923 158 : if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2924 49 : merge = build_int_cst (TREE_TYPE (vectype), 0);
2925 109 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2926 : {
2927 : REAL_VALUE_TYPE r;
2928 : long tmp[6];
2929 763 : for (int j = 0; j < 6; ++j)
2930 654 : tmp[j] = 0;
2931 109 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2932 109 : merge = build_real (TREE_TYPE (vectype), r);
2933 : }
2934 : else
2935 0 : gcc_unreachable ();
2936 158 : merge = build_vector_from_val (vectype, merge);
2937 158 : return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
2938 : }
2939 :
2940 : /* Return the corresponding else value for an else value constant
2941 : ELSVAL with type TYPE. */
2942 :
2943 : tree
2944 1962 : vect_get_mask_load_else (int elsval, tree type)
2945 : {
2946 1962 : tree els;
2947 1962 : if (elsval == MASK_LOAD_ELSE_UNDEFINED)
2948 : {
2949 0 : tree tmp = create_tmp_var (type);
2950 : /* No need to warn about anything. */
2951 0 : TREE_NO_WARNING (tmp) = 1;
2952 0 : els = get_or_create_ssa_default_def (cfun, tmp);
2953 : }
2954 1962 : else if (elsval == MASK_LOAD_ELSE_M1)
2955 0 : els = build_minus_one_cst (type);
2956 1962 : else if (elsval == MASK_LOAD_ELSE_ZERO)
2957 1962 : els = build_zero_cst (type);
2958 : else
2959 0 : gcc_unreachable ();
2960 :
2961 1962 : return els;
2962 : }
2963 :
2964 : /* Build a gather load call while vectorizing STMT_INFO. Insert new
2965 : instructions before GSI and add them to VEC_STMT. GS_INFO describes
2966 : the gather load operation. If the load is conditional, MASK is the
2967 : vectorized condition, otherwise MASK is null. PTR is the base
2968 : pointer and OFFSET is the vectorized offset. */
2969 :
2970 : static gimple *
2971 349 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
2972 : slp_tree slp_node, tree vectype,
2973 : gimple_stmt_iterator *gsi, tree decl,
2974 : tree ptr, tree offset, tree mask)
2975 : {
2976 349 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
2977 349 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
2978 349 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2979 349 : /* ptrtype */ arglist = TREE_CHAIN (arglist);
2980 349 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2981 349 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2982 349 : tree scaletype = TREE_VALUE (arglist);
2983 349 : tree var;
2984 349 : gcc_checking_assert (types_compatible_p (srctype, rettype)
2985 : && (!mask
2986 : || TREE_CODE (masktype) == INTEGER_TYPE
2987 : || types_compatible_p (srctype, masktype)));
2988 :
2989 349 : tree op = offset;
2990 349 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
2991 : {
2992 103 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
2993 : TYPE_VECTOR_SUBPARTS (idxtype)));
2994 103 : var = vect_get_new_ssa_name (idxtype, vect_simple_var);
2995 103 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
2996 103 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
2997 103 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
2998 103 : op = var;
2999 : }
3000 :
3001 349 : tree src_op = NULL_TREE;
3002 349 : tree mask_op = NULL_TREE;
3003 349 : if (mask)
3004 : {
3005 191 : if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
3006 : {
3007 191 : tree utype, optype = TREE_TYPE (mask);
3008 191 : if (VECTOR_TYPE_P (masktype)
3009 191 : || TYPE_MODE (masktype) == TYPE_MODE (optype))
3010 : utype = masktype;
3011 : else
3012 6 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3013 191 : var = vect_get_new_ssa_name (utype, vect_scalar_var);
3014 191 : tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
3015 191 : gassign *new_stmt
3016 191 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3017 191 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3018 191 : mask_arg = var;
3019 191 : if (!useless_type_conversion_p (masktype, utype))
3020 : {
3021 6 : gcc_assert (TYPE_PRECISION (utype)
3022 : <= TYPE_PRECISION (masktype));
3023 6 : var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3024 6 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3025 6 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3026 6 : mask_arg = var;
3027 : }
3028 191 : src_op = build_zero_cst (srctype);
3029 191 : mask_op = mask_arg;
3030 : }
3031 : else
3032 : {
3033 : src_op = mask;
3034 : mask_op = mask;
3035 : }
3036 : }
3037 : else
3038 : {
3039 158 : src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
3040 158 : mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
3041 : }
3042 :
3043 349 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3044 349 : gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
3045 : mask_op, scale);
3046 :
3047 349 : if (!useless_type_conversion_p (vectype, rettype))
3048 : {
3049 52 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
3050 : TYPE_VECTOR_SUBPARTS (rettype)));
3051 52 : op = vect_get_new_ssa_name (rettype, vect_simple_var);
3052 52 : gimple_call_set_lhs (new_stmt, op);
3053 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3054 52 : op = build1 (VIEW_CONVERT_EXPR, vectype, op);
3055 52 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
3056 : }
3057 :
3058 349 : return new_stmt;
3059 : }
3060 :
3061 : /* Build a scatter store call while vectorizing STMT_INFO. Insert new
3062 : instructions before GSI. GS_INFO describes the scatter store operation.
3063 : PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
3064 : vectorized data to store.
3065 : If the store is conditional, MASK is the vectorized condition, otherwise
3066 : MASK is null. */
3067 :
3068 : static gimple *
3069 161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
3070 : slp_tree slp_node,
3071 : gimple_stmt_iterator *gsi,
3072 : tree decl,
3073 : tree ptr, tree offset, tree oprnd, tree mask)
3074 : {
3075 161 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
3076 161 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
3077 161 : /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
3078 161 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3079 161 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3080 161 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3081 161 : tree scaletype = TREE_VALUE (arglist);
3082 161 : gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
3083 : && TREE_CODE (rettype) == VOID_TYPE);
3084 :
3085 161 : tree mask_arg = NULL_TREE;
3086 161 : if (mask)
3087 : {
3088 110 : mask_arg = mask;
3089 110 : tree optype = TREE_TYPE (mask_arg);
3090 110 : tree utype;
3091 110 : if (TYPE_MODE (masktype) == TYPE_MODE (optype))
3092 : utype = masktype;
3093 : else
3094 8 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3095 110 : tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
3096 110 : mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
3097 110 : gassign *new_stmt
3098 110 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3099 110 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3100 110 : mask_arg = var;
3101 110 : if (!useless_type_conversion_p (masktype, utype))
3102 : {
3103 8 : gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
3104 8 : tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3105 8 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3106 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3107 8 : mask_arg = var;
3108 : }
3109 : }
3110 : else
3111 : {
3112 51 : mask_arg = build_int_cst (masktype, -1);
3113 51 : mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
3114 : }
3115 :
3116 161 : tree src = oprnd;
3117 161 : if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
3118 : {
3119 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
3120 : TYPE_VECTOR_SUBPARTS (srctype)));
3121 0 : tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
3122 0 : src = build1 (VIEW_CONVERT_EXPR, srctype, src);
3123 0 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
3124 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3125 0 : src = var;
3126 : }
3127 :
3128 161 : tree op = offset;
3129 161 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3130 : {
3131 16 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3132 : TYPE_VECTOR_SUBPARTS (idxtype)));
3133 16 : tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3134 16 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3135 16 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3136 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3137 16 : op = var;
3138 : }
3139 :
3140 161 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3141 161 : gcall *new_stmt
3142 161 : = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
3143 161 : return new_stmt;
3144 : }
3145 :
3146 : /* Prepare the base and offset in GS_INFO for vectorization.
3147 : Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
3148 : to the vectorized offset argument for the first copy of STMT_INFO.
3149 : STMT_INFO is the statement described by GS_INFO and LOOP is the
3150 : containing loop. */
3151 :
3152 : static void
3153 1239 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
3154 : tree *dataref_ptr, vec<tree> *vec_offset)
3155 : {
3156 1239 : gimple_seq stmts = NULL;
3157 1239 : *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
3158 : &stmts, true, NULL_TREE);
3159 1239 : if (stmts != NULL)
3160 : {
3161 1006 : basic_block new_bb;
3162 1006 : edge pe = loop_preheader_edge (loop);
3163 1006 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3164 1006 : gcc_assert (!new_bb);
3165 : }
3166 1239 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
3167 1239 : }
3168 :
3169 : /* Prepare to implement a grouped or strided load or store using
3170 : the gather load or scatter store operation described by GS_INFO.
3171 : STMT_INFO is the load or store statement.
3172 :
3173 : Set *DR_STEP to the amount that should be added to pointer base address
3174 : to get to the next iteration's base address.
3175 : Set *DR_BUMP to the amount that should be added to the base
3176 : address after each copy of the vectorized statement in a grouped read.
3177 : Set *VEC_OFFSET to an invariant offset vector in which element I has the
3178 : value I * DR_STEP / SCALE. */
3179 :
3180 : static void
3181 0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
3182 : tree vectype, tree offset_vectype,
3183 : loop_vec_info loop_vinfo,
3184 : gimple_stmt_iterator *gsi,
3185 : tree *dr_step, tree *dr_bump,
3186 : tree *vec_offset)
3187 : {
3188 0 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
3189 :
3190 0 : tree dr_step_temp
3191 0 : = size_binop (MULT_EXPR,
3192 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3193 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
3194 0 : *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
3195 0 : ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
3196 0 : : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
3197 : true, GSI_SAME_STMT);
3198 0 : tree bump = size_binop (MULT_EXPR,
3199 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3200 : size_int (TYPE_VECTOR_SUBPARTS (vectype)));
3201 0 : *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
3202 :
3203 0 : internal_fn ifn
3204 0 : = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
3205 0 : if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
3206 : {
3207 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
3208 : unshare_expr (DR_STEP (dr)));
3209 0 : return;
3210 : }
3211 :
3212 : /* The offset given in GS_INFO can have pointer type, so use the element
3213 : type of the vector instead. */
3214 0 : tree offset_type = TREE_TYPE (offset_vectype);
3215 :
3216 : /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3217 0 : tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
3218 : ssize_int (SLP_TREE_GS_SCALE (node)));
3219 0 : step = fold_convert (offset_type, step);
3220 :
3221 : /* Create {0, X, X*2, X*3, ...}. */
3222 0 : tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
3223 : build_zero_cst (offset_type), step);
3224 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
3225 : }
3226 :
3227 : /* Return the amount that should be added to a vector pointer, represented by
3228 : DR_INFO, to increment to the next vectorized iteration. */
3229 :
3230 : static tree
3231 716386 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
3232 : vect_memory_access_type memory_access_type)
3233 : {
3234 716386 : if (memory_access_type == VMAT_INVARIANT)
3235 0 : return size_zero_node;
3236 :
3237 716386 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3238 :
3239 : /* For BB SLP there is no next iteration. */
3240 716386 : if (!loop_vinfo)
3241 581250 : return build_zero_cst (sizetype);
3242 :
3243 135136 : tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
3244 :
3245 : /* gather/scatter never reach here. */
3246 135136 : gcc_assert (!mat_gather_scatter_p (memory_access_type));
3247 :
3248 135136 : tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
3249 :
3250 135136 : return fold_build2 (MULT_EXPR, sizetype, iv_increment,
3251 : fold_convert (sizetype, step));
3252 : }
3253 :
3254 : /* Return the amount that should be added to a vector pointer to move
3255 : to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3256 : being vectorized and MEMORY_ACCESS_TYPE describes the type of
3257 : vectorization. */
3258 :
3259 : static tree
3260 716898 : vect_get_data_ptr_bump (vec_info *vinfo,
3261 : dr_vec_info *dr_info, tree aggr_type,
3262 : vect_memory_access_type memory_access_type)
3263 : {
3264 716898 : if (memory_access_type == VMAT_INVARIANT)
3265 0 : return size_zero_node;
3266 :
3267 716898 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3268 : /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
3269 : should never be needed. */
3270 135648 : if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
3271 : return NULL_TREE;
3272 :
3273 716898 : tree iv_step = TYPE_SIZE_UNIT (aggr_type);
3274 716898 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
3275 716898 : if (tree_int_cst_sgn (step) == -1)
3276 2840 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
3277 : return iv_step;
3278 : }
3279 :
3280 : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3281 :
3282 : static bool
3283 140 : vectorizable_bswap (vec_info *vinfo,
3284 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3285 : slp_tree slp_node,
3286 : slp_tree *slp_op,
3287 : tree vectype_in, stmt_vector_for_cost *cost_vec)
3288 : {
3289 140 : tree vectype = SLP_TREE_VECTYPE (slp_node);
3290 140 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3291 :
3292 140 : if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
3293 : {
3294 0 : if (dump_enabled_p ())
3295 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3296 : "mismatched vector sizes %T and %T\n",
3297 : vectype_in, vectype);
3298 : return false;
3299 : }
3300 :
3301 140 : tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
3302 140 : if (! char_vectype)
3303 : return false;
3304 :
3305 140 : poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
3306 140 : unsigned word_bytes;
3307 140 : if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
3308 : return false;
3309 :
3310 : /* The encoding uses one stepped pattern for each byte in the word. */
3311 140 : vec_perm_builder elts (num_bytes, word_bytes, 3);
3312 700 : for (unsigned i = 0; i < 3; ++i)
3313 2532 : for (unsigned j = 0; j < word_bytes; ++j)
3314 2112 : elts.quick_push ((i + 1) * word_bytes - j - 1);
3315 :
3316 140 : vec_perm_indices indices (elts, 1, num_bytes);
3317 140 : machine_mode vmode = TYPE_MODE (char_vectype);
3318 140 : if (!can_vec_perm_const_p (vmode, vmode, indices))
3319 : return false;
3320 :
3321 57 : if (cost_vec)
3322 : {
3323 41 : if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
3324 : {
3325 0 : if (dump_enabled_p ())
3326 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3327 : "incompatible vector types for invariants\n");
3328 : return false;
3329 : }
3330 :
3331 41 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3332 41 : DUMP_VECT_SCOPE ("vectorizable_bswap");
3333 41 : record_stmt_cost (cost_vec,
3334 : 1, vector_stmt, slp_node, 0, vect_prologue);
3335 41 : record_stmt_cost (cost_vec,
3336 41 : vect_get_num_copies (vinfo, slp_node),
3337 : vec_perm, slp_node, 0, vect_body);
3338 41 : return true;
3339 : }
3340 :
3341 16 : tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
3342 :
3343 : /* Transform. */
3344 16 : vec<tree> vec_oprnds = vNULL;
3345 16 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
3346 : /* Arguments are ready. create the new vector stmt. */
3347 16 : unsigned i;
3348 16 : tree vop;
3349 48 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
3350 : {
3351 16 : gimple *new_stmt;
3352 16 : tree tem = make_ssa_name (char_vectype);
3353 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3354 : char_vectype, vop));
3355 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3356 16 : tree tem2 = make_ssa_name (char_vectype);
3357 16 : new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
3358 : tem, tem, bswap_vconst);
3359 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3360 16 : tem = make_ssa_name (vectype);
3361 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3362 : vectype, tem2));
3363 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3364 16 : slp_node->push_vec_def (new_stmt);
3365 : }
3366 :
3367 16 : vec_oprnds.release ();
3368 16 : return true;
3369 140 : }
3370 :
3371 : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3372 : integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3373 : in a single step. On success, store the binary pack code in
3374 : *CONVERT_CODE. */
3375 :
3376 : static bool
3377 197 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
3378 : code_helper *convert_code)
3379 : {
3380 394 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
3381 394 : || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
3382 : return false;
3383 :
3384 87 : code_helper code;
3385 87 : int multi_step_cvt = 0;
3386 87 : auto_vec <tree, 8> interm_types;
3387 130 : if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
3388 : &code, &multi_step_cvt, &interm_types)
3389 87 : || multi_step_cvt)
3390 : return false;
3391 :
3392 44 : *convert_code = code;
3393 44 : return true;
3394 87 : }
3395 :
3396 : /* Function vectorizable_call.
3397 :
3398 : Check if STMT_INFO performs a function call that can be vectorized.
3399 : If COST_VEC is passed, calculate costs but don't change anything,
3400 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
3401 : it, and insert it at GSI.
3402 : Return true if STMT_INFO is vectorizable in this way. */
3403 :
3404 : static bool
3405 2641998 : vectorizable_call (vec_info *vinfo,
3406 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3407 : slp_tree slp_node,
3408 : stmt_vector_for_cost *cost_vec)
3409 : {
3410 2641998 : gcall *stmt;
3411 2641998 : tree vec_dest;
3412 2641998 : tree scalar_dest;
3413 2641998 : tree vec_oprnd0 = NULL_TREE;
3414 2641998 : tree vectype_out, vectype_in;
3415 2641998 : poly_uint64 nunits_in;
3416 2641998 : poly_uint64 nunits_out;
3417 2641998 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3418 2641998 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3419 2641998 : tree fndecl, new_temp, rhs_type;
3420 2641998 : enum vect_def_type dt[5]
3421 : = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
3422 : vect_unknown_def_type, vect_unknown_def_type };
3423 2641998 : tree vectypes[ARRAY_SIZE (dt)] = {};
3424 2641998 : slp_tree slp_op[ARRAY_SIZE (dt)] = {};
3425 2641998 : auto_vec<tree, 8> vargs;
3426 2641998 : enum { NARROW, NONE, WIDEN } modifier;
3427 2641998 : size_t i, nargs;
3428 2641998 : tree clz_ctz_arg1 = NULL_TREE;
3429 :
3430 2641998 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3431 : return false;
3432 :
3433 2641998 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3434 244447 : && cost_vec)
3435 : return false;
3436 :
3437 : /* Is STMT_INFO a vectorizable call? */
3438 2668355 : stmt = dyn_cast <gcall *> (stmt_info->stmt);
3439 26357 : if (!stmt)
3440 : return false;
3441 :
3442 26357 : if (gimple_call_internal_p (stmt)
3443 26357 : && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3444 16535 : || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3445 : /* Handled by vectorizable_load and vectorizable_store. */
3446 : return false;
3447 :
3448 22565 : if (gimple_call_lhs (stmt) == NULL_TREE
3449 22565 : || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3450 : return false;
3451 :
3452 22559 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
3453 :
3454 22559 : vectype_out = SLP_TREE_VECTYPE (slp_node);
3455 :
3456 : /* Process function arguments. */
3457 22559 : rhs_type = NULL_TREE;
3458 22559 : vectype_in = NULL_TREE;
3459 22559 : nargs = gimple_call_num_args (stmt);
3460 :
3461 : /* Bail out if the function has more than four arguments, we do not have
3462 : interesting builtin functions to vectorize with more than two arguments
3463 : except for fma (cond_fma has more). No arguments is also not good. */
3464 22559 : if (nargs == 0 || nargs > 5)
3465 : return false;
3466 :
3467 : /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3468 22479 : combined_fn cfn = gimple_call_combined_fn (stmt);
3469 22479 : if (cfn == CFN_GOMP_SIMD_LANE)
3470 : {
3471 3203 : nargs = 0;
3472 3203 : rhs_type = unsigned_type_node;
3473 : }
3474 : /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3475 : argument just says whether it is well-defined at zero or not and what
3476 : value should be returned for it. */
3477 22479 : if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
3478 : {
3479 168 : nargs = 1;
3480 168 : clz_ctz_arg1 = gimple_call_arg (stmt, 1);
3481 : }
3482 :
3483 22479 : int mask_opno = -1;
3484 22479 : if (internal_fn_p (cfn))
3485 : {
3486 : /* We can only handle direct internal masked calls here,
3487 : vectorizable_simd_clone_call is for the rest. */
3488 19117 : if (cfn == CFN_MASK_CALL)
3489 : return false;
3490 18963 : mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
3491 : }
3492 :
3493 69534 : for (i = 0; i < nargs; i++)
3494 : {
3495 49273 : if ((int) i == mask_opno)
3496 : {
3497 7694 : if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
3498 : &slp_op[i], &dt[i], &vectypes[i]))
3499 : return false;
3500 7694 : continue;
3501 : }
3502 :
3503 41579 : if (!vect_is_simple_use (vinfo, slp_node,
3504 : i, &slp_op[i], &dt[i], &vectypes[i]))
3505 : {
3506 0 : if (dump_enabled_p ())
3507 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3508 : "use not simple.\n");
3509 : return false;
3510 : }
3511 :
3512 : /* We can only handle calls with arguments of the same type. */
3513 41579 : tree op = gimple_call_arg (stmt, i);
3514 41579 : if (rhs_type
3515 41579 : && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3516 : {
3517 2064 : if (dump_enabled_p ())
3518 222 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3519 : "argument types differ.\n");
3520 : return false;
3521 : }
3522 39515 : if (!rhs_type)
3523 19122 : rhs_type = TREE_TYPE (op);
3524 :
3525 39515 : if (!vectype_in)
3526 20382 : vectype_in = vectypes[i];
3527 19133 : else if (vectypes[i]
3528 19133 : && !types_compatible_p (vectypes[i], vectype_in))
3529 : {
3530 0 : if (dump_enabled_p ())
3531 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3532 : "argument vector types differ.\n");
3533 : return false;
3534 : }
3535 : }
3536 : /* If all arguments are external or constant defs, infer the vector type
3537 : from the scalar type. */
3538 20261 : if (!vectype_in)
3539 6117 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
3540 20261 : if (!cost_vec)
3541 4209 : gcc_assert (vectype_in);
3542 16052 : if (!vectype_in)
3543 : {
3544 1121 : if (dump_enabled_p ())
3545 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3546 : "no vectype for scalar type %T\n", rhs_type);
3547 :
3548 : return false;
3549 : }
3550 :
3551 38280 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3552 19140 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
3553 : {
3554 12 : if (dump_enabled_p ())
3555 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3556 : "mixed mask and nonmask vector types\n");
3557 : return false;
3558 : }
3559 :
3560 19128 : if (vect_emulated_vector_p (vectype_in)
3561 19128 : || vect_emulated_vector_p (vectype_out))
3562 : {
3563 20 : if (dump_enabled_p ())
3564 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3565 : "use emulated vector type for call\n");
3566 : return false;
3567 : }
3568 :
3569 : /* FORNOW */
3570 19108 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3571 19108 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3572 19108 : if (known_eq (nunits_in * 2, nunits_out))
3573 : modifier = NARROW;
3574 18474 : else if (known_eq (nunits_out, nunits_in))
3575 : modifier = NONE;
3576 50 : else if (known_eq (nunits_out * 2, nunits_in))
3577 : modifier = WIDEN;
3578 : else
3579 : return false;
3580 :
3581 : /* We only handle functions that do not read or clobber memory. */
3582 38216 : if (gimple_vuse (stmt))
3583 : {
3584 1743 : if (dump_enabled_p ())
3585 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3586 : "function reads from or writes to memory.\n");
3587 : return false;
3588 : }
3589 :
3590 : /* For now, we only vectorize functions if a target specific builtin
3591 : is available. TODO -- in some cases, it might be profitable to
3592 : insert the calls for pieces of the vector, in order to be able
3593 : to vectorize other operations in the loop. */
3594 17365 : fndecl = NULL_TREE;
3595 17365 : internal_fn ifn = IFN_LAST;
3596 17365 : tree callee = gimple_call_fndecl (stmt);
3597 :
3598 : /* First try using an internal function. */
3599 17365 : code_helper convert_code = MAX_TREE_CODES;
3600 17365 : if (cfn != CFN_LAST
3601 17365 : && (modifier == NONE
3602 209 : || (modifier == NARROW
3603 197 : && simple_integer_narrowing (vectype_out, vectype_in,
3604 : &convert_code))))
3605 16341 : ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3606 : vectype_in);
3607 :
3608 : /* Check if the operation traps. */
3609 17365 : bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
3610 17365 : if (could_trap && cost_vec && loop_vinfo)
3611 : {
3612 : /* If the operation can trap it must be conditional, otherwise fail. */
3613 474 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3614 474 : ? ifn : get_conditional_internal_fn (ifn));
3615 474 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3616 474 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
3617 : {
3618 : /* We assume that BB SLP fills all lanes, so no inactive lanes can
3619 : cause issues. */
3620 84 : if ((cond_fn == IFN_LAST
3621 56 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3622 : OPTIMIZE_FOR_SPEED))
3623 140 : && (cond_len_fn == IFN_LAST
3624 56 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3625 : OPTIMIZE_FOR_SPEED)))
3626 : {
3627 84 : if (dump_enabled_p ())
3628 10 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3629 : "can't use a fully-masked loop because no"
3630 : " conditional operation is available.\n");
3631 84 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3632 : }
3633 : }
3634 : }
3635 :
3636 : /* If that fails, try asking for a target-specific built-in function. */
3637 17365 : if (ifn == IFN_LAST)
3638 : {
3639 9896 : if (cfn != CFN_LAST)
3640 9037 : fndecl = targetm.vectorize.builtin_vectorized_function
3641 9037 : (cfn, vectype_out, vectype_in);
3642 859 : else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
3643 24 : fndecl = targetm.vectorize.builtin_md_vectorized_function
3644 24 : (callee, vectype_out, vectype_in);
3645 : }
3646 :
3647 17365 : if (ifn == IFN_LAST && !fndecl)
3648 : {
3649 9516 : if (cfn == CFN_GOMP_SIMD_LANE
3650 3203 : && SLP_TREE_LANES (slp_node) == 1
3651 3203 : && loop_vinfo
3652 3203 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3653 3203 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3654 15922 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3655 3203 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3656 : {
3657 : /* We can handle IFN_GOMP_SIMD_LANE by returning a
3658 : { 0, 1, 2, ... vf - 1 } vector. */
3659 3203 : gcc_assert (nargs == 0);
3660 : }
3661 6313 : else if (modifier == NONE
3662 6313 : && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3663 5954 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3664 5901 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
3665 5848 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
3666 140 : return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
3667 140 : slp_op, vectype_in, cost_vec);
3668 : else
3669 : {
3670 6173 : if (dump_enabled_p ())
3671 262 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3672 : "function is not vectorizable.\n");
3673 : return false;
3674 : }
3675 : }
3676 :
3677 11052 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
3678 11052 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3679 11052 : ? ifn : get_conditional_internal_fn (ifn));
3680 11052 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3681 11052 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3682 9154 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
3683 11052 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
3684 11052 : if (cost_vec) /* transformation not required. */
3685 : {
3686 21750 : for (i = 0; i < nargs; ++i)
3687 14891 : if (!vect_maybe_update_slp_op_vectype (slp_op[i],
3688 14891 : vectypes[i]
3689 : ? vectypes[i] : vectype_in))
3690 : {
3691 0 : if (dump_enabled_p ())
3692 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3693 : "incompatible vector types for invariants\n");
3694 : return false;
3695 : }
3696 6859 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3697 6859 : DUMP_VECT_SCOPE ("vectorizable_call");
3698 6859 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
3699 :
3700 6859 : if (loop_vinfo
3701 5901 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
3702 4036 : && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3703 : {
3704 2558 : if (reduc_idx >= 0
3705 1631 : && (cond_fn == IFN_LAST
3706 1631 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3707 : OPTIMIZE_FOR_SPEED))
3708 2570 : && (cond_len_fn == IFN_LAST
3709 12 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3710 : OPTIMIZE_FOR_SPEED)))
3711 : {
3712 12 : if (dump_enabled_p ())
3713 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3714 : "can't use a fully-masked loop because no"
3715 : " conditional operation is available.\n");
3716 12 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3717 : }
3718 : else
3719 : {
3720 2546 : tree scalar_mask = NULL_TREE;
3721 2546 : if (mask_opno >= 0)
3722 2546 : scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3723 2546 : if (cond_len_fn != IFN_LAST
3724 2546 : && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3725 : OPTIMIZE_FOR_SPEED))
3726 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3727 : 1);
3728 : else
3729 2546 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3730 : scalar_mask);
3731 : }
3732 : }
3733 6859 : return true;
3734 : }
3735 :
3736 : /* Transform. */
3737 :
3738 4193 : if (dump_enabled_p ())
3739 420 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3740 :
3741 : /* Handle def. */
3742 4193 : scalar_dest = gimple_call_lhs (stmt);
3743 4193 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3744 :
3745 4193 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3746 3253 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
3747 4193 : unsigned int vect_nargs = nargs;
3748 4193 : if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3749 : {
3750 0 : ifn = cond_len_fn;
3751 : /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3752 0 : vect_nargs += 2;
3753 : /* But unless there's a mask argument already we need that
3754 : as well, and an else value. */
3755 0 : if (mask_opno == -1)
3756 0 : vect_nargs += 2;
3757 : }
3758 4193 : else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
3759 : {
3760 0 : ifn = cond_fn;
3761 0 : vect_nargs += 2;
3762 : }
3763 4193 : int len_opno = internal_fn_len_index (ifn);
3764 4193 : if (clz_ctz_arg1)
3765 59 : ++vect_nargs;
3766 :
3767 4193 : if (modifier == NONE || ifn != IFN_LAST)
3768 : {
3769 4161 : tree prev_res = NULL_TREE;
3770 4161 : vargs.safe_grow (vect_nargs, true);
3771 4161 : auto_vec<vec<tree> > vec_defs (nargs);
3772 :
3773 : /* Build argument list for the vectorized call. */
3774 4161 : if (cfn == CFN_GOMP_SIMD_LANE)
3775 : {
3776 3304 : for (i = 0; i < nvectors; ++i)
3777 : {
3778 : /* ??? For multi-lane SLP we'd need to build
3779 : { 0, 0, .., 1, 1, ... }. */
3780 1706 : tree cst = build_index_vector (vectype_out,
3781 : i * nunits_out, 1);
3782 1706 : tree new_var
3783 1706 : = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3784 1706 : gimple *init_stmt = gimple_build_assign (new_var, cst);
3785 1706 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
3786 1706 : new_temp = make_ssa_name (vec_dest);
3787 1706 : gimple *new_stmt = gimple_build_assign (new_temp, new_var);
3788 1706 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3789 1706 : slp_node->push_vec_def (new_stmt);
3790 : }
3791 : }
3792 : else
3793 : {
3794 2563 : vec<tree> vec_oprnds0;
3795 2563 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3796 2563 : vec_oprnds0 = vec_defs[0];
3797 :
3798 : /* Arguments are ready. Create the new vector stmt. */
3799 5279 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3800 : {
3801 2716 : int varg = 0;
3802 : /* Add the mask if necessary. */
3803 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3804 2718 : && internal_fn_mask_index (ifn) != -1)
3805 : {
3806 0 : gcc_assert (internal_fn_mask_index (ifn) == varg);
3807 0 : if (masked_loop_p)
3808 : {
3809 0 : unsigned int vec_num = vec_oprnds0.length ();
3810 0 : vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
3811 : masks, vec_num,
3812 : vectype_out, i);
3813 : }
3814 : else
3815 : {
3816 0 : tree mask_vectype = truth_type_for (vectype_out);
3817 0 : vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
3818 : stmt_info,
3819 : mask_vectype);
3820 : }
3821 : }
3822 2716 : size_t k;
3823 9948 : for (k = 0; k < nargs; k++)
3824 : {
3825 7232 : vec<tree> vec_oprndsk = vec_defs[k];
3826 7232 : vargs[varg++] = vec_oprndsk[i];
3827 : }
3828 : /* Add the else value if necessary. */
3829 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3830 2718 : && internal_fn_else_index (ifn) != -1)
3831 : {
3832 0 : gcc_assert (internal_fn_else_index (ifn) == varg);
3833 0 : if (reduc_idx >= 0)
3834 0 : vargs[varg++] = vargs[reduc_idx + 1];
3835 : else
3836 : {
3837 0 : auto else_value = targetm.preferred_else_value
3838 0 : (ifn, vectype_out, varg - 1, &vargs[1]);
3839 0 : vargs[varg++] = else_value;
3840 : }
3841 : }
3842 2716 : if (clz_ctz_arg1)
3843 59 : vargs[varg++] = clz_ctz_arg1;
3844 :
3845 2716 : gimple *new_stmt;
3846 2716 : if (modifier == NARROW)
3847 : {
3848 : /* We don't define any narrowing conditional functions
3849 : at present. */
3850 0 : gcc_assert (mask_opno < 0);
3851 0 : tree half_res = make_ssa_name (vectype_in);
3852 0 : gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3853 0 : gimple_call_set_lhs (call, half_res);
3854 0 : gimple_call_set_nothrow (call, true);
3855 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3856 0 : if ((i & 1) == 0)
3857 : {
3858 0 : prev_res = half_res;
3859 0 : continue;
3860 : }
3861 0 : new_temp = make_ssa_name (vec_dest);
3862 0 : new_stmt = vect_gimple_build (new_temp, convert_code,
3863 : prev_res, half_res);
3864 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3865 : }
3866 : else
3867 : {
3868 2716 : if (len_opno >= 0 && len_loop_p)
3869 : {
3870 0 : unsigned int vec_num = vec_oprnds0.length ();
3871 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
3872 : vec_num, vectype_out, i, 1, true);
3873 0 : signed char biasval
3874 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3875 0 : tree bias = build_int_cst (intQI_type_node, biasval);
3876 0 : vargs[len_opno] = len;
3877 0 : vargs[len_opno + 1] = bias;
3878 : }
3879 2716 : else if (mask_opno >= 0 && masked_loop_p)
3880 : {
3881 36 : unsigned int vec_num = vec_oprnds0.length ();
3882 36 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
3883 : vec_num, vectype_out, i);
3884 36 : vargs[mask_opno]
3885 72 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
3886 36 : vargs[mask_opno], gsi);
3887 : }
3888 :
3889 2716 : gcall *call;
3890 2716 : if (ifn != IFN_LAST)
3891 2635 : call = gimple_build_call_internal_vec (ifn, vargs);
3892 : else
3893 81 : call = gimple_build_call_vec (fndecl, vargs);
3894 2716 : new_temp = make_ssa_name (vec_dest, call);
3895 2716 : gimple_call_set_lhs (call, new_temp);
3896 2716 : gimple_call_set_nothrow (call, true);
3897 2716 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3898 2716 : new_stmt = call;
3899 : }
3900 2716 : slp_node->push_vec_def (new_stmt);
3901 : }
3902 : }
3903 :
3904 11031 : for (i = 0; i < nargs; i++)
3905 : {
3906 6870 : vec<tree> vec_oprndsi = vec_defs[i];
3907 6870 : vec_oprndsi.release ();
3908 : }
3909 4161 : }
3910 32 : else if (modifier == NARROW)
3911 : {
3912 32 : auto_vec<vec<tree> > vec_defs (nargs);
3913 : /* We don't define any narrowing conditional functions at present. */
3914 32 : gcc_assert (mask_opno < 0);
3915 :
3916 : /* Build argument list for the vectorized call. */
3917 32 : vargs.create (nargs * 2);
3918 :
3919 32 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3920 32 : vec<tree> vec_oprnds0 = vec_defs[0];
3921 :
3922 : /* Arguments are ready. Create the new vector stmt. */
3923 64 : for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3924 : {
3925 32 : size_t k;
3926 32 : vargs.truncate (0);
3927 96 : for (k = 0; k < nargs; k++)
3928 : {
3929 32 : vec<tree> vec_oprndsk = vec_defs[k];
3930 32 : vargs.quick_push (vec_oprndsk[i]);
3931 32 : vargs.quick_push (vec_oprndsk[i + 1]);
3932 : }
3933 32 : gcall *call;
3934 32 : if (ifn != IFN_LAST)
3935 : call = gimple_build_call_internal_vec (ifn, vargs);
3936 : else
3937 32 : call = gimple_build_call_vec (fndecl, vargs);
3938 32 : new_temp = make_ssa_name (vec_dest, call);
3939 32 : gimple_call_set_lhs (call, new_temp);
3940 32 : gimple_call_set_nothrow (call, true);
3941 32 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3942 32 : slp_node->push_vec_def (call);
3943 : }
3944 :
3945 64 : for (i = 0; i < nargs; i++)
3946 : {
3947 32 : vec<tree> vec_oprndsi = vec_defs[i];
3948 32 : vec_oprndsi.release ();
3949 : }
3950 32 : }
3951 : else
3952 : /* No current target implements this case. */
3953 : return false;
3954 :
3955 4193 : vargs.release ();
3956 :
3957 4193 : return true;
3958 2641998 : }
3959 :
3960 :
3961 : struct simd_call_arg_info
3962 : {
3963 : tree vectype;
3964 : tree op;
3965 : HOST_WIDE_INT linear_step;
3966 : enum vect_def_type dt;
3967 : unsigned int align;
3968 : bool simd_lane_linear;
3969 : };
3970 :
3971 : /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3972 : is linear within simd lane (but not within whole loop), note it in
3973 : *ARGINFO. */
3974 :
3975 : static void
3976 15 : vect_simd_lane_linear (tree op, class loop *loop,
3977 : struct simd_call_arg_info *arginfo)
3978 : {
3979 15 : gimple *def_stmt = SSA_NAME_DEF_STMT (op);
3980 :
3981 15 : if (!is_gimple_assign (def_stmt)
3982 15 : || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
3983 27 : || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
3984 : return;
3985 :
3986 12 : tree base = gimple_assign_rhs1 (def_stmt);
3987 12 : HOST_WIDE_INT linear_step = 0;
3988 12 : tree v = gimple_assign_rhs2 (def_stmt);
3989 48 : while (TREE_CODE (v) == SSA_NAME)
3990 : {
3991 36 : tree t;
3992 36 : def_stmt = SSA_NAME_DEF_STMT (v);
3993 36 : if (is_gimple_assign (def_stmt))
3994 24 : switch (gimple_assign_rhs_code (def_stmt))
3995 : {
3996 0 : case PLUS_EXPR:
3997 0 : t = gimple_assign_rhs2 (def_stmt);
3998 0 : if (linear_step || TREE_CODE (t) != INTEGER_CST)
3999 : return;
4000 0 : base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
4001 0 : v = gimple_assign_rhs1 (def_stmt);
4002 0 : continue;
4003 12 : case MULT_EXPR:
4004 12 : t = gimple_assign_rhs2 (def_stmt);
4005 12 : if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
4006 : return;
4007 12 : linear_step = tree_to_shwi (t);
4008 12 : v = gimple_assign_rhs1 (def_stmt);
4009 12 : continue;
4010 12 : CASE_CONVERT:
4011 12 : t = gimple_assign_rhs1 (def_stmt);
4012 12 : if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
4013 12 : || (TYPE_PRECISION (TREE_TYPE (v))
4014 12 : < TYPE_PRECISION (TREE_TYPE (t))))
4015 : return;
4016 12 : if (!linear_step)
4017 0 : linear_step = 1;
4018 12 : v = t;
4019 12 : continue;
4020 : default:
4021 : return;
4022 : }
4023 12 : else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
4024 12 : && loop->simduid
4025 12 : && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
4026 24 : && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
4027 : == loop->simduid))
4028 : {
4029 12 : if (!linear_step)
4030 0 : linear_step = 1;
4031 12 : arginfo->linear_step = linear_step;
4032 12 : arginfo->op = base;
4033 12 : arginfo->simd_lane_linear = true;
4034 12 : return;
4035 : }
4036 : }
4037 : }
4038 :
4039 : /* Function vectorizable_simd_clone_call.
4040 :
4041 : Check if STMT_INFO performs a function call that can be vectorized
4042 : by calling a simd clone of the function.
4043 : If COST_VEC is passed, calculate costs but don't change anything,
4044 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
4045 : it, and insert it at GSI.
4046 : Return true if STMT_INFO is vectorizable in this way. */
4047 :
4048 : static bool
4049 2631247 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
4050 : gimple_stmt_iterator *gsi,
4051 : slp_tree slp_node,
4052 : stmt_vector_for_cost *cost_vec)
4053 : {
4054 2631247 : tree vec_dest;
4055 2631247 : tree scalar_dest;
4056 2631247 : tree vec_oprnd0 = NULL_TREE;
4057 2631247 : tree vectype;
4058 2631247 : poly_uint64 nunits;
4059 2631247 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
4060 2631247 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
4061 2631247 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
4062 2631247 : tree fndecl, new_temp;
4063 2631247 : int j;
4064 2631247 : auto_vec<simd_call_arg_info> arginfo;
4065 2631247 : vec<tree> vargs = vNULL;
4066 2631247 : size_t i, nargs;
4067 2631247 : tree rtype, ratype;
4068 2631247 : vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
4069 2631247 : int masked_call_offset = 0;
4070 :
4071 : /* Is STMT a vectorizable call? */
4072 2631247 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
4073 16787 : if (!stmt)
4074 : return false;
4075 :
4076 16787 : fndecl = gimple_call_fndecl (stmt);
4077 16787 : if (fndecl == NULL_TREE
4078 16787 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
4079 : {
4080 220 : fndecl = gimple_call_arg (stmt, 0);
4081 220 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
4082 220 : fndecl = TREE_OPERAND (fndecl, 0);
4083 220 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
4084 : masked_call_offset = 1;
4085 : }
4086 16567 : if (fndecl == NULL_TREE)
4087 : return false;
4088 :
4089 6299 : struct cgraph_node *node = cgraph_node::get (fndecl);
4090 6299 : if (node == NULL || node->simd_clones == NULL)
4091 : return false;
4092 :
4093 1585 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4094 : return false;
4095 :
4096 1585 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4097 0 : && cost_vec)
4098 : return false;
4099 :
4100 1585 : if (gimple_call_lhs (stmt)
4101 1585 : && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
4102 : return false;
4103 :
4104 1585 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
4105 :
4106 1585 : vectype = SLP_TREE_VECTYPE (slp_node);
4107 :
4108 2632832 : if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
4109 : return false;
4110 :
4111 : /* Process function arguments. */
4112 1585 : nargs = gimple_call_num_args (stmt) - masked_call_offset;
4113 :
4114 : /* Bail out if the function has zero arguments. */
4115 1585 : if (nargs == 0)
4116 : return false;
4117 :
4118 1521 : vect_simd_clone_data _data;
4119 1521 : vect_simd_clone_data &data = slp_node->get_data (_data);
4120 1521 : vec<tree>& simd_clone_info = data.simd_clone_info;
4121 1521 : arginfo.reserve (nargs, true);
4122 1521 : auto_vec<slp_tree> slp_op;
4123 1521 : slp_op.safe_grow_cleared (nargs);
4124 :
4125 5836 : for (i = 0; i < nargs; i++)
4126 : {
4127 2794 : simd_call_arg_info thisarginfo;
4128 2794 : affine_iv iv;
4129 2794 : tree op;
4130 :
4131 2794 : thisarginfo.linear_step = 0;
4132 2794 : thisarginfo.align = 0;
4133 2794 : thisarginfo.op = NULL_TREE;
4134 2794 : thisarginfo.simd_lane_linear = false;
4135 :
4136 5588 : int op_no = vect_slp_child_index_for_operand (stmt_info,
4137 2794 : i + masked_call_offset);
4138 5588 : if (!vect_is_simple_use (vinfo, slp_node,
4139 2794 : op_no, &op, &slp_op[i],
4140 : &thisarginfo.dt, &thisarginfo.vectype)
4141 2794 : || thisarginfo.dt == vect_uninitialized_def)
4142 : {
4143 0 : if (dump_enabled_p ())
4144 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4145 : "use not simple.\n");
4146 0 : return false;
4147 : }
4148 :
4149 2794 : if (thisarginfo.dt == vect_constant_def
4150 2794 : || thisarginfo.dt == vect_external_def)
4151 : {
4152 : /* With SLP we determine the vector type of constants/externals
4153 : at analysis time, handling conflicts via
4154 : vect_maybe_update_slp_op_vectype. At transform time
4155 : we have a vector type recorded for SLP. */
4156 783 : gcc_assert (cost_vec
4157 : || thisarginfo.vectype != NULL_TREE);
4158 : if (cost_vec)
4159 652 : thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
4160 652 : TREE_TYPE (op),
4161 : slp_node);
4162 : }
4163 : else
4164 2011 : gcc_assert (thisarginfo.vectype != NULL_TREE);
4165 :
4166 : /* For linear arguments, the analyze phase should have saved
4167 : the base and step. */
4168 2663 : if (!cost_vec
4169 1586 : && i * 3 + 4 <= simd_clone_info.length ()
4170 2873 : && simd_clone_info[i * 3 + 2])
4171 : {
4172 118 : thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
4173 118 : thisarginfo.op = simd_clone_info[i * 3 + 1];
4174 118 : thisarginfo.simd_lane_linear
4175 118 : = (simd_clone_info[i * 3 + 3] == boolean_true_node);
4176 : /* If loop has been peeled for alignment, we need to adjust it. */
4177 118 : tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
4178 118 : tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
4179 118 : if (n1 != n2 && !thisarginfo.simd_lane_linear)
4180 : {
4181 0 : tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
4182 0 : tree step = simd_clone_info[i * 3 + 2];
4183 0 : tree opt = TREE_TYPE (thisarginfo.op);
4184 0 : bias = fold_convert (TREE_TYPE (step), bias);
4185 0 : bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
4186 0 : thisarginfo.op
4187 0 : = fold_build2 (POINTER_TYPE_P (opt)
4188 : ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
4189 : thisarginfo.op, bias);
4190 : }
4191 : }
4192 2676 : else if (cost_vec
4193 2001 : && thisarginfo.dt != vect_constant_def
4194 1874 : && thisarginfo.dt != vect_external_def
4195 1349 : && loop_vinfo
4196 1344 : && SLP_TREE_LANES (slp_node) == 1
4197 1320 : && TREE_CODE (op) == SSA_NAME
4198 2640 : && simple_iv (loop, loop_containing_stmt (stmt), op,
4199 : &iv, false)
4200 2888 : && tree_fits_shwi_p (iv.step))
4201 : {
4202 212 : thisarginfo.linear_step = tree_to_shwi (iv.step);
4203 212 : thisarginfo.op = iv.base;
4204 : }
4205 2464 : else if ((thisarginfo.dt == vect_constant_def
4206 2464 : || thisarginfo.dt == vect_external_def)
4207 783 : && SLP_TREE_LANES (slp_node) == 1
4208 2770 : && POINTER_TYPE_P (TREE_TYPE (op)))
4209 86 : thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
4210 : /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4211 : linear too. */
4212 2794 : if (SLP_TREE_LANES (slp_node) == 1
4213 2271 : && POINTER_TYPE_P (TREE_TYPE (op))
4214 196 : && !thisarginfo.linear_step
4215 112 : && cost_vec
4216 58 : && thisarginfo.dt != vect_constant_def
4217 58 : && thisarginfo.dt != vect_external_def
4218 15 : && loop_vinfo
4219 2809 : && TREE_CODE (op) == SSA_NAME)
4220 15 : vect_simd_lane_linear (op, loop, &thisarginfo);
4221 :
4222 2794 : if (!vectype)
4223 12 : vectype = thisarginfo.vectype;
4224 2794 : arginfo.quick_push (thisarginfo);
4225 : }
4226 :
4227 1521 : poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
4228 1521 : unsigned group_size = SLP_TREE_LANES (slp_node);
4229 1521 : unsigned int badness = 0;
4230 1521 : unsigned int badness_inbranch = 0;
4231 1521 : struct cgraph_node *bestn = NULL;
4232 1521 : struct cgraph_node *bestn_inbranch = NULL;
4233 1521 : if (!cost_vec)
4234 358 : bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4235 358 : ? data.clone_inbranch : data.clone);
4236 : else
4237 6563 : for (struct cgraph_node *n = node->simd_clones; n != NULL;
4238 5400 : n = n->simdclone->next_clone)
4239 : {
4240 5400 : unsigned int this_badness = 0;
4241 5400 : unsigned int num_calls;
4242 : /* The number of arguments in the call and the number of parameters in
4243 : the simdclone should match. However, when the simdclone is
4244 : 'inbranch', it could have one more parameter than nargs when using
4245 : an inbranch simdclone to call a non-inbranch call, either in a
4246 : non-masked loop using a all true constant mask, or inside a masked
4247 : loop using it's mask. */
4248 5400 : size_t simd_nargs = n->simdclone->nargs;
4249 5400 : if (!masked_call_offset && n->simdclone->inbranch)
4250 2635 : simd_nargs--;
4251 5400 : if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
4252 : &num_calls)
4253 1948 : || (!n->simdclone->inbranch && (masked_call_offset > 0))
4254 1764 : || (nargs != simd_nargs))
4255 3636 : continue;
4256 1764 : if (num_calls != 1)
4257 1162 : this_badness += floor_log2 (num_calls) * 4096;
4258 1764 : if (n->simdclone->inbranch)
4259 729 : this_badness += 8192;
4260 :
4261 : /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
4262 : mode, which for targets that use it will determine what ISA we can
4263 : vectorize this code with. */
4264 1764 : machine_mode vector_mode = vinfo->vector_mode;
4265 1764 : if (vectype)
4266 1764 : vector_mode = TYPE_MODE (vectype);
4267 1764 : int target_badness = targetm.simd_clone.usable (n, vector_mode);
4268 1764 : if (target_badness < 0)
4269 380 : continue;
4270 1384 : this_badness += target_badness * 512;
4271 4184 : for (i = 0; i < nargs; i++)
4272 : {
4273 3048 : switch (n->simdclone->args[i].arg_type)
4274 : {
4275 2118 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4276 2118 : if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
4277 : /* Vector mask arguments are not supported. */
4278 : i = -1;
4279 2110 : else if (!useless_type_conversion_p
4280 2110 : (n->simdclone->args[i].orig_type,
4281 2110 : TREE_TYPE (gimple_call_arg (stmt,
4282 : i + masked_call_offset))))
4283 : i = -1;
4284 2110 : else if (arginfo[i].dt == vect_constant_def
4285 2003 : || arginfo[i].dt == vect_external_def
4286 4049 : || arginfo[i].linear_step)
4287 399 : this_badness += 64;
4288 : break;
4289 310 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4290 310 : if ((arginfo[i].dt != vect_constant_def
4291 145 : && arginfo[i].dt != vect_external_def)
4292 410 : || SLP_TREE_LANES (slp_node) != 1)
4293 : i = -1;
4294 : break;
4295 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4296 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4297 324 : if (arginfo[i].dt == vect_constant_def
4298 324 : || arginfo[i].dt == vect_external_def
4299 324 : || (arginfo[i].linear_step
4300 324 : != n->simdclone->args[i].linear_step))
4301 : i = -1;
4302 : break;
4303 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4304 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4305 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4306 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4307 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4308 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4309 : /* FORNOW */
4310 : i = -1;
4311 : break;
4312 296 : case SIMD_CLONE_ARG_TYPE_MASK:
4313 296 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4314 264 : && n->simdclone->mask_mode != VOIDmode)
4315 : i = -1;
4316 : /* While we can create a traditional data vector from
4317 : an incoming integer mode mask we have no good way to
4318 : force generate an integer mode mask from a traditional
4319 : boolean vector input. */
4320 296 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4321 296 : && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4322 : i = -1;
4323 290 : else if (n->simdclone->mask_mode == VOIDmode
4324 : /* FORNOW we only have partial support for vector-type
4325 : masks that can't hold all of simdlen. */
4326 554 : && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
4327 429 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
4328 : /* Verify we can compute the mask argument. */
4329 111 : || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
4330 111 : arginfo[i].vectype)))
4331 : i = -1;
4332 125 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4333 : /* FORNOW we only have partial support for
4334 : integer-type masks that represent the same number
4335 : of lanes as the vectorized mask inputs. */
4336 151 : && maybe_ne (exact_div (n->simdclone->simdlen,
4337 : n->simdclone->args[i].linear_step),
4338 26 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4339 : i = -1;
4340 107 : else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4341 107 : && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4342 8 : this_badness += 2048;
4343 : break;
4344 : }
4345 18 : if (i == (size_t) -1)
4346 : break;
4347 2800 : if (n->simdclone->args[i].alignment > arginfo[i].align)
4348 : {
4349 : i = -1;
4350 : break;
4351 : }
4352 2800 : if (arginfo[i].align)
4353 110 : this_badness += (exact_log2 (arginfo[i].align)
4354 160 : - exact_log2 (n->simdclone->args[i].alignment));
4355 : }
4356 1384 : if (i == (size_t) -1)
4357 248 : continue;
4358 1136 : if (masked_call_offset == 0
4359 1029 : && n->simdclone->inbranch
4360 311 : && n->simdclone->nargs > nargs)
4361 : {
4362 311 : gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
4363 : SIMD_CLONE_ARG_TYPE_MASK);
4364 : /* Penalize using a masked SIMD clone in a non-masked loop, that is
4365 : not in a branch, as we'd have to construct an all-true mask. */
4366 311 : this_badness += 64;
4367 : }
4368 1136 : if (bestn == NULL || this_badness < badness)
4369 : {
4370 815 : bestn = n;
4371 815 : badness = this_badness;
4372 : }
4373 1136 : if (n->simdclone->inbranch
4374 418 : && (bestn_inbranch == NULL || this_badness < badness_inbranch))
4375 : {
4376 5400 : bestn_inbranch = n;
4377 5400 : badness_inbranch = this_badness;
4378 : }
4379 : }
4380 :
4381 1521 : if (bestn == NULL)
4382 : return false;
4383 :
4384 835 : fndecl = bestn->decl;
4385 835 : nunits = bestn->simdclone->simdlen;
4386 835 : int ncopies = vector_unroll_factor (vf * group_size, nunits);
4387 :
4388 : /* If the function isn't const, only allow it in simd loops where user
4389 : has asserted that at least nunits consecutive iterations can be
4390 : performed using SIMD instructions. */
4391 830 : if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
4392 996 : && gimple_vuse (stmt))
4393 : return false;
4394 :
4395 : /* ncopies is the number of SIMD clone calls we create, since simdlen
4396 : is not necessarily matching nunits of the vector types used, track
4397 : that in ncopies_in. */
4398 835 : int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
4399 :
4400 : /* Sanity check: make sure that at least one copy of the vectorized stmt
4401 : needs to be generated. */
4402 835 : gcc_assert (ncopies >= 1);
4403 :
4404 835 : if (cost_vec) /* transformation not required. */
4405 : {
4406 1578 : for (unsigned i = 0; i < nargs; ++i)
4407 1101 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
4408 : {
4409 0 : if (dump_enabled_p ())
4410 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4411 : "incompatible vector types for invariants\n");
4412 : return false;
4413 : }
4414 :
4415 477 : if (!bestn_inbranch && loop_vinfo)
4416 : {
4417 270 : if (dump_enabled_p ()
4418 270 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
4419 193 : dump_printf_loc (MSG_NOTE, vect_location,
4420 : "can't use a fully-masked loop because no"
4421 : " masked simd clone was available.\n");
4422 270 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4423 : }
4424 :
4425 : /* When the original call is pure or const but the SIMD ABI dictates
4426 : an aggregate return we will have to use a virtual definition and
4427 : in a loop eventually even need to add a virtual PHI. That's
4428 : not straight-forward so allow to fix this up via renaming. */
4429 477 : if (gimple_call_lhs (stmt)
4430 471 : && !gimple_vdef (stmt)
4431 844 : && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
4432 33 : vinfo->any_known_not_updated_vssa = true;
4433 : /* ??? For SLP code-gen we end up inserting after the last
4434 : vector argument def rather than at the original call position
4435 : so automagic virtual operand updating doesn't work. */
4436 954 : if (gimple_vuse (stmt))
4437 147 : vinfo->any_known_not_updated_vssa = true;
4438 :
4439 477 : data.clone = bestn;
4440 477 : data.clone_inbranch = bestn_inbranch;
4441 :
4442 477 : simd_clone_info.safe_push (NULL_TREE);
4443 1715 : for (i = 0;
4444 2642 : i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
4445 : {
4446 1238 : if (loop_vinfo
4447 1232 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
4448 458 : && (bestn_inbranch->simdclone->args[i].arg_type
4449 : == SIMD_CLONE_ARG_TYPE_MASK))
4450 : {
4451 162 : if (masked_call_offset)
4452 : /* When there is an explicit mask we require the
4453 : number of elements to match up. */
4454 49 : vect_record_loop_mask (loop_vinfo,
4455 : &LOOP_VINFO_MASKS (loop_vinfo),
4456 : ncopies_in, vectype, NULL_TREE);
4457 : else
4458 : {
4459 : /* When there is no explicit mask on the call we have
4460 : more relaxed requirements. */
4461 113 : tree masktype;
4462 113 : poly_uint64 callee_nelements;
4463 113 : if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
4464 : {
4465 12 : callee_nelements
4466 12 : = exact_div (bestn_inbranch->simdclone->simdlen,
4467 : bestn_inbranch->simdclone->args[i].linear_step);
4468 12 : masktype = get_related_vectype_for_scalar_type
4469 12 : (vinfo->vector_mode, TREE_TYPE (vectype),
4470 : callee_nelements);
4471 : }
4472 : else
4473 : {
4474 101 : masktype = bestn_inbranch->simdclone->args[i].vector_type;
4475 : /* The aarch64 port will add custom attributes to types
4476 : for SVE simdclones which make the types different. We
4477 : should use canonincal types for masks within the
4478 : vectorizer, hence we construct the related vectype
4479 : here. */
4480 101 : masktype
4481 : = build_truth_vector_type_for_mode
4482 101 : (TYPE_VECTOR_SUBPARTS (masktype),
4483 101 : TYPE_MODE (masktype));
4484 101 : callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
4485 : }
4486 113 : auto o = vector_unroll_factor (nunits, callee_nelements);
4487 113 : vect_record_loop_mask (loop_vinfo,
4488 : &LOOP_VINFO_MASKS (loop_vinfo),
4489 : ncopies * o, masktype, NULL_TREE);
4490 : }
4491 : }
4492 1076 : else if ((bestn->simdclone->args[i].arg_type
4493 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4494 969 : || (bestn->simdclone->args[i].arg_type
4495 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
4496 958 : || (bestn_inbranch
4497 352 : && ((bestn_inbranch->simdclone->args[i].arg_type
4498 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4499 352 : || (bestn_inbranch->simdclone->args[i].arg_type
4500 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
4501 : {
4502 118 : simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
4503 118 : simd_clone_info.safe_push (arginfo[i].op);
4504 202 : tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
4505 202 : ? size_type_node : TREE_TYPE (arginfo[i].op));
4506 118 : tree ls = build_int_cst (lst, arginfo[i].linear_step);
4507 118 : simd_clone_info.safe_push (ls);
4508 118 : tree sll = (arginfo[i].simd_lane_linear
4509 118 : ? boolean_true_node : boolean_false_node);
4510 118 : simd_clone_info.safe_push (sll);
4511 : }
4512 : }
4513 :
4514 477 : SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
4515 477 : slp_node->data = new vect_simd_clone_data (std::move (_data));
4516 477 : DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4517 : /* ??? We're confused by calls w/o LHS. */
4518 477 : if (SLP_TREE_VECTYPE (slp_node))
4519 471 : vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
4520 477 : return true;
4521 : }
4522 :
4523 : /* Transform. */
4524 :
4525 358 : if (dump_enabled_p ())
4526 242 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
4527 :
4528 : /* Handle def. */
4529 358 : scalar_dest = gimple_call_lhs (stmt);
4530 358 : vec_dest = NULL_TREE;
4531 358 : rtype = NULL_TREE;
4532 358 : ratype = NULL_TREE;
4533 358 : if (scalar_dest)
4534 : {
4535 352 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
4536 352 : rtype = TREE_TYPE (TREE_TYPE (fndecl));
4537 352 : if (TREE_CODE (rtype) == ARRAY_TYPE)
4538 : {
4539 9 : ratype = rtype;
4540 9 : rtype = TREE_TYPE (ratype);
4541 : }
4542 : }
4543 :
4544 716 : auto_vec<vec<tree> > vec_oprnds;
4545 358 : auto_vec<unsigned> vec_oprnds_i;
4546 358 : vec_oprnds_i.safe_grow_cleared (nargs, true);
4547 358 : vec_oprnds.reserve_exact (nargs);
4548 358 : vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
4549 1181 : for (j = 0; j < ncopies; ++j)
4550 : {
4551 465 : poly_uint64 callee_nelements;
4552 465 : poly_uint64 caller_nelements;
4553 : /* Build argument list for the vectorized call. */
4554 465 : if (j == 0)
4555 358 : vargs.create (nargs);
4556 : else
4557 107 : vargs.truncate (0);
4558 :
4559 1568 : for (i = 0; i < nargs; i++)
4560 : {
4561 1103 : unsigned int k, l, m, o;
4562 1103 : tree atype;
4563 1103 : tree op = gimple_call_arg (stmt, i + masked_call_offset);
4564 1103 : switch (bestn->simdclone->args[i].arg_type)
4565 : {
4566 814 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4567 814 : atype = bestn->simdclone->args[i].vector_type;
4568 814 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4569 814 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4570 814 : o = vector_unroll_factor (nunits, callee_nelements);
4571 1858 : for (m = j * o; m < (j + 1) * o; m++)
4572 : {
4573 1044 : if (known_lt (callee_nelements, caller_nelements))
4574 : {
4575 516 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
4576 258 : if (!constant_multiple_p (caller_nelements,
4577 : callee_nelements, &k))
4578 0 : gcc_unreachable ();
4579 :
4580 258 : gcc_assert ((k & (k - 1)) == 0);
4581 258 : if (m == 0)
4582 : {
4583 57 : vec_oprnds_i[i] = 0;
4584 57 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4585 : }
4586 : else
4587 : {
4588 201 : vec_oprnd0 = arginfo[i].op;
4589 201 : if ((m & (k - 1)) == 0)
4590 72 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4591 : }
4592 258 : arginfo[i].op = vec_oprnd0;
4593 258 : vec_oprnd0
4594 258 : = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
4595 258 : bitsize_int (prec),
4596 258 : bitsize_int ((m & (k - 1)) * prec));
4597 258 : gassign *new_stmt
4598 258 : = gimple_build_assign (make_ssa_name (atype),
4599 : vec_oprnd0);
4600 258 : vect_finish_stmt_generation (vinfo, stmt_info,
4601 : new_stmt, gsi);
4602 258 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4603 : }
4604 : else
4605 : {
4606 786 : if (!constant_multiple_p (callee_nelements,
4607 : caller_nelements, &k))
4608 0 : gcc_unreachable ();
4609 786 : gcc_assert ((k & (k - 1)) == 0);
4610 786 : vec<constructor_elt, va_gc> *ctor_elts;
4611 786 : if (k != 1)
4612 12 : vec_alloc (ctor_elts, k);
4613 : else
4614 774 : ctor_elts = NULL;
4615 810 : for (l = 0; l < k; l++)
4616 : {
4617 798 : if (m == 0 && l == 0)
4618 : {
4619 450 : vec_oprnds_i[i] = 0;
4620 450 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4621 : }
4622 : else
4623 348 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4624 798 : arginfo[i].op = vec_oprnd0;
4625 798 : if (k == 1)
4626 : break;
4627 24 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
4628 : vec_oprnd0);
4629 : }
4630 786 : if (k == 1)
4631 774 : if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
4632 : atype))
4633 : {
4634 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
4635 : vec_oprnd0);
4636 0 : gassign *new_stmt
4637 0 : = gimple_build_assign (make_ssa_name (atype),
4638 : vec_oprnd0);
4639 0 : vect_finish_stmt_generation (vinfo, stmt_info,
4640 : new_stmt, gsi);
4641 0 : vargs.safe_push (gimple_get_lhs (new_stmt));
4642 : }
4643 : else
4644 774 : vargs.safe_push (vec_oprnd0);
4645 : else
4646 : {
4647 12 : vec_oprnd0 = build_constructor (atype, ctor_elts);
4648 12 : gassign *new_stmt
4649 12 : = gimple_build_assign (make_ssa_name (atype),
4650 : vec_oprnd0);
4651 12 : vect_finish_stmt_generation (vinfo, stmt_info,
4652 : new_stmt, gsi);
4653 12 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4654 : }
4655 : }
4656 : }
4657 : break;
4658 66 : case SIMD_CLONE_ARG_TYPE_MASK:
4659 66 : if (bestn->simdclone->mask_mode == VOIDmode)
4660 : {
4661 60 : atype = bestn->simdclone->args[i].vector_type;
4662 60 : tree elt_type = TREE_TYPE (atype);
4663 60 : tree one = fold_convert (elt_type, integer_one_node);
4664 60 : tree zero = fold_convert (elt_type, integer_zero_node);
4665 60 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4666 60 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4667 60 : o = vector_unroll_factor (nunits, callee_nelements);
4668 120 : for (m = j * o; m < (j + 1) * o; m++)
4669 : {
4670 60 : if (maybe_lt (callee_nelements, caller_nelements))
4671 : {
4672 : /* The mask type has fewer elements than simdlen. */
4673 :
4674 : /* FORNOW */
4675 0 : gcc_unreachable ();
4676 : }
4677 60 : else if (known_eq (callee_nelements, caller_nelements))
4678 : {
4679 : /* The SIMD clone function has the same number of
4680 : elements as the current function. */
4681 60 : if (m == 0)
4682 60 : vec_oprnds_i[i] = 0;
4683 60 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4684 60 : if (loop_vinfo
4685 60 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4686 : {
4687 0 : vec_loop_masks *loop_masks
4688 : = &LOOP_VINFO_MASKS (loop_vinfo);
4689 0 : tree loop_mask
4690 0 : = vect_get_loop_mask (loop_vinfo, gsi,
4691 : loop_masks, ncopies_in,
4692 0 : vectype, j);
4693 0 : vec_oprnd0
4694 0 : = prepare_vec_mask (loop_vinfo,
4695 0 : TREE_TYPE (loop_mask),
4696 : loop_mask, vec_oprnd0,
4697 : gsi);
4698 0 : loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
4699 : loop_mask });
4700 :
4701 : }
4702 60 : vec_oprnd0
4703 60 : = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
4704 : build_vector_from_val (atype, one),
4705 : build_vector_from_val (atype, zero));
4706 60 : gassign *new_stmt
4707 60 : = gimple_build_assign (make_ssa_name (atype),
4708 : vec_oprnd0);
4709 60 : vect_finish_stmt_generation (vinfo, stmt_info,
4710 : new_stmt, gsi);
4711 60 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4712 : }
4713 : else
4714 : {
4715 : /* The mask type has more elements than simdlen. */
4716 :
4717 : /* FORNOW */
4718 0 : gcc_unreachable ();
4719 : }
4720 : }
4721 : }
4722 6 : else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4723 : {
4724 6 : atype = bestn->simdclone->args[i].vector_type;
4725 6 : poly_uint64 atype_subparts
4726 6 : = exact_div (bestn->simdclone->simdlen,
4727 : bestn->simdclone->args[i].linear_step);
4728 6 : o = bestn->simdclone->args[i].linear_step;
4729 12 : for (m = j * o; m < (j + 1) * o; m++)
4730 : {
4731 6 : if (m == 0)
4732 6 : vec_oprnds_i[i] = 0;
4733 6 : if (maybe_lt (atype_subparts,
4734 6 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4735 : {
4736 : /* The mask argument has fewer elements than the
4737 : input vector. */
4738 : /* FORNOW */
4739 0 : gcc_unreachable ();
4740 : }
4741 6 : else if (known_eq (atype_subparts,
4742 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4743 : {
4744 6 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4745 6 : if (loop_vinfo
4746 6 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4747 : {
4748 1 : vec_loop_masks *loop_masks
4749 : = &LOOP_VINFO_MASKS (loop_vinfo);
4750 1 : tree loop_mask
4751 1 : = vect_get_loop_mask (loop_vinfo, gsi,
4752 : loop_masks, ncopies_in,
4753 : vectype, j);
4754 1 : vec_oprnd0
4755 1 : = prepare_vec_mask (loop_vinfo,
4756 1 : TREE_TYPE (loop_mask),
4757 : loop_mask, vec_oprnd0,
4758 : gsi);
4759 : }
4760 : /* The vector mask argument matches the input
4761 : in the number of lanes, but not necessarily
4762 : in the mode. */
4763 6 : tree st = lang_hooks.types.type_for_mode
4764 6 : (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
4765 6 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
4766 : vec_oprnd0);
4767 6 : gassign *new_stmt
4768 6 : = gimple_build_assign (make_ssa_name (st),
4769 : vec_oprnd0);
4770 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4771 : new_stmt, gsi);
4772 6 : if (!types_compatible_p (atype, st))
4773 : {
4774 6 : new_stmt
4775 6 : = gimple_build_assign (make_ssa_name (atype),
4776 : NOP_EXPR,
4777 : gimple_assign_lhs
4778 : (new_stmt));
4779 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4780 : new_stmt, gsi);
4781 : }
4782 6 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4783 : }
4784 : else
4785 : {
4786 : /* The mask argument has more elements than the
4787 : input vector. */
4788 : /* FORNOW */
4789 0 : gcc_unreachable ();
4790 : }
4791 : }
4792 : }
4793 : else
4794 0 : gcc_unreachable ();
4795 : break;
4796 102 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4797 102 : vargs.safe_push (op);
4798 102 : break;
4799 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4800 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4801 121 : if (j == 0)
4802 : {
4803 118 : gimple_seq stmts;
4804 118 : arginfo[i].op
4805 118 : = force_gimple_operand (unshare_expr (arginfo[i].op),
4806 : &stmts, true, NULL_TREE);
4807 118 : if (stmts != NULL)
4808 : {
4809 0 : basic_block new_bb;
4810 0 : edge pe = loop_preheader_edge (loop);
4811 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4812 0 : gcc_assert (!new_bb);
4813 : }
4814 118 : if (arginfo[i].simd_lane_linear)
4815 : {
4816 6 : vargs.safe_push (arginfo[i].op);
4817 6 : break;
4818 : }
4819 112 : tree phi_res = copy_ssa_name (op);
4820 112 : gphi *new_phi = create_phi_node (phi_res, loop->header);
4821 112 : add_phi_arg (new_phi, arginfo[i].op,
4822 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
4823 112 : enum tree_code code
4824 196 : = POINTER_TYPE_P (TREE_TYPE (op))
4825 112 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4826 196 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4827 196 : ? sizetype : TREE_TYPE (op);
4828 112 : poly_widest_int cst
4829 112 : = wi::mul (bestn->simdclone->args[i].linear_step,
4830 112 : ncopies * nunits);
4831 112 : tree tcst = wide_int_to_tree (type, cst);
4832 112 : tree phi_arg = copy_ssa_name (op);
4833 112 : gassign *new_stmt
4834 112 : = gimple_build_assign (phi_arg, code, phi_res, tcst);
4835 112 : gimple_stmt_iterator si = gsi_after_labels (loop->header);
4836 112 : gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4837 112 : add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4838 : UNKNOWN_LOCATION);
4839 112 : arginfo[i].op = phi_res;
4840 112 : vargs.safe_push (phi_res);
4841 112 : }
4842 : else
4843 : {
4844 3 : enum tree_code code
4845 6 : = POINTER_TYPE_P (TREE_TYPE (op))
4846 3 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4847 6 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4848 6 : ? sizetype : TREE_TYPE (op);
4849 3 : poly_widest_int cst
4850 3 : = wi::mul (bestn->simdclone->args[i].linear_step,
4851 3 : j * nunits);
4852 3 : tree tcst = wide_int_to_tree (type, cst);
4853 3 : new_temp = make_ssa_name (TREE_TYPE (op));
4854 3 : gassign *new_stmt
4855 6 : = gimple_build_assign (new_temp, code,
4856 3 : arginfo[i].op, tcst);
4857 3 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4858 3 : vargs.safe_push (new_temp);
4859 3 : }
4860 : break;
4861 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4862 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4863 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4864 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4865 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4866 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4867 0 : default:
4868 0 : gcc_unreachable ();
4869 : }
4870 : }
4871 :
4872 465 : if (masked_call_offset == 0
4873 399 : && bestn->simdclone->inbranch
4874 13 : && bestn->simdclone->nargs > nargs)
4875 : {
4876 13 : unsigned long m, o;
4877 13 : size_t mask_i = bestn->simdclone->nargs - 1;
4878 13 : tree mask;
4879 13 : gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
4880 : SIMD_CLONE_ARG_TYPE_MASK);
4881 :
4882 13 : tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
4883 13 : tree mask_vectype;
4884 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4885 : {
4886 2 : callee_nelements = exact_div (bestn->simdclone->simdlen,
4887 : bestn->simdclone->args[i].linear_step);
4888 2 : mask_vectype = get_related_vectype_for_scalar_type
4889 2 : (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
4890 : }
4891 : else
4892 : {
4893 11 : mask_vectype = mask_argtype;
4894 11 : callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
4895 : }
4896 13 : o = vector_unroll_factor (nunits, callee_nelements);
4897 26 : for (m = j * o; m < (j + 1) * o; m++)
4898 : {
4899 13 : if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4900 : {
4901 1 : vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
4902 1 : mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
4903 : ncopies * o, mask_vectype, m);
4904 : }
4905 : else
4906 12 : mask = vect_build_all_ones_mask (vinfo, stmt_info,
4907 : mask_argtype);
4908 :
4909 13 : gassign *new_stmt;
4910 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4911 : {
4912 : /* This means we are dealing with integer mask modes.
4913 : First convert to an integer type with the same size as
4914 : the current vector type. */
4915 2 : unsigned HOST_WIDE_INT intermediate_size
4916 2 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
4917 2 : tree mid_int_type =
4918 2 : build_nonstandard_integer_type (intermediate_size, 1);
4919 2 : mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
4920 2 : new_stmt
4921 2 : = gimple_build_assign (make_ssa_name (mid_int_type),
4922 : mask);
4923 2 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4924 : /* Then zero-extend to the mask mode. */
4925 2 : mask = fold_build1 (NOP_EXPR, mask_argtype,
4926 : gimple_get_lhs (new_stmt));
4927 : }
4928 11 : else if (bestn->simdclone->mask_mode == VOIDmode)
4929 11 : mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
4930 : build_one_cst (mask_argtype),
4931 : build_zero_cst (mask_argtype));
4932 : else
4933 0 : gcc_unreachable ();
4934 :
4935 13 : new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
4936 : mask);
4937 13 : vect_finish_stmt_generation (vinfo, stmt_info,
4938 : new_stmt, gsi);
4939 13 : mask = gimple_assign_lhs (new_stmt);
4940 13 : vargs.safe_push (mask);
4941 : }
4942 : }
4943 :
4944 465 : gcall *new_call = gimple_build_call_vec (fndecl, vargs);
4945 465 : if (vec_dest)
4946 : {
4947 459 : gcc_assert (ratype
4948 : || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
4949 459 : if (ratype)
4950 15 : new_temp = create_tmp_var (ratype);
4951 444 : else if (useless_type_conversion_p (vectype, rtype))
4952 424 : new_temp = make_ssa_name (vec_dest, new_call);
4953 : else
4954 20 : new_temp = make_ssa_name (rtype, new_call);
4955 459 : gimple_call_set_lhs (new_call, new_temp);
4956 : }
4957 465 : vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
4958 465 : gimple *new_stmt = new_call;
4959 :
4960 465 : if (vec_dest)
4961 : {
4962 459 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
4963 : {
4964 19 : unsigned int k, l;
4965 38 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
4966 38 : poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
4967 19 : k = vector_unroll_factor (nunits,
4968 : TYPE_VECTOR_SUBPARTS (vectype));
4969 19 : gcc_assert ((k & (k - 1)) == 0);
4970 69 : for (l = 0; l < k; l++)
4971 : {
4972 50 : tree t;
4973 50 : if (ratype)
4974 : {
4975 42 : t = build_fold_addr_expr (new_temp);
4976 42 : t = build2 (MEM_REF, vectype, t,
4977 42 : build_int_cst (TREE_TYPE (t), l * bytes));
4978 : }
4979 : else
4980 8 : t = build3 (BIT_FIELD_REF, vectype, new_temp,
4981 8 : bitsize_int (prec), bitsize_int (l * prec));
4982 50 : new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
4983 50 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4984 :
4985 50 : SLP_TREE_VEC_DEFS (slp_node)
4986 50 : .quick_push (gimple_assign_lhs (new_stmt));
4987 : }
4988 :
4989 19 : if (ratype)
4990 15 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
4991 19 : continue;
4992 19 : }
4993 440 : else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
4994 : {
4995 16 : unsigned int k;
4996 16 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
4997 16 : TYPE_VECTOR_SUBPARTS (rtype), &k))
4998 0 : gcc_unreachable ();
4999 16 : gcc_assert ((k & (k - 1)) == 0);
5000 16 : if ((j & (k - 1)) == 0)
5001 8 : vec_alloc (ret_ctor_elts, k);
5002 16 : if (ratype)
5003 : {
5004 0 : unsigned int m, o;
5005 0 : o = vector_unroll_factor (nunits,
5006 : TYPE_VECTOR_SUBPARTS (rtype));
5007 0 : for (m = 0; m < o; m++)
5008 : {
5009 0 : tree tem = build4 (ARRAY_REF, rtype, new_temp,
5010 0 : size_int (m), NULL_TREE, NULL_TREE);
5011 0 : new_stmt = gimple_build_assign (make_ssa_name (rtype),
5012 : tem);
5013 0 : vect_finish_stmt_generation (vinfo, stmt_info,
5014 : new_stmt, gsi);
5015 0 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
5016 : gimple_assign_lhs (new_stmt));
5017 : }
5018 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5019 : }
5020 : else
5021 16 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
5022 16 : if ((j & (k - 1)) != k - 1)
5023 8 : continue;
5024 8 : vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
5025 8 : new_stmt
5026 8 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5027 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5028 :
5029 8 : SLP_TREE_VEC_DEFS (slp_node)
5030 8 : .quick_push (gimple_assign_lhs (new_stmt));
5031 8 : continue;
5032 8 : }
5033 424 : else if (ratype)
5034 : {
5035 0 : tree t = build_fold_addr_expr (new_temp);
5036 0 : t = build2 (MEM_REF, vectype, t,
5037 0 : build_int_cst (TREE_TYPE (t), 0));
5038 0 : new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
5039 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5040 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5041 : }
5042 424 : else if (!useless_type_conversion_p (vectype, rtype))
5043 : {
5044 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
5045 0 : new_stmt
5046 0 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5047 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5048 : }
5049 : }
5050 :
5051 430 : if (gimple_get_lhs (new_stmt))
5052 424 : SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
5053 : }
5054 :
5055 1151 : for (i = 0; i < nargs; ++i)
5056 : {
5057 793 : vec<tree> oprndsi = vec_oprnds[i];
5058 793 : oprndsi.release ();
5059 : }
5060 358 : vargs.release ();
5061 :
5062 : /* Mark the clone as no longer being a candidate for GC. */
5063 358 : bestn->gc_candidate = false;
5064 :
5065 358 : return true;
5066 1521 : }
5067 :
5068 :
5069 : /* Function vect_gen_widened_results_half
5070 :
5071 : Create a vector stmt whose code, type, number of arguments, and result
5072 : variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5073 : VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5074 : In the case that CODE is a CALL_EXPR, this means that a call to DECL
5075 : needs to be created (DECL is a function-decl of a target-builtin).
5076 : STMT_INFO is the original scalar stmt that we are vectorizing. */
5077 :
5078 : static gimple *
5079 32308 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
5080 : tree vec_oprnd0, tree vec_oprnd1, int op_type,
5081 : tree vec_dest, gimple_stmt_iterator *gsi,
5082 : stmt_vec_info stmt_info)
5083 : {
5084 32308 : gimple *new_stmt;
5085 32308 : tree new_temp;
5086 :
5087 : /* Generate half of the widened result: */
5088 32308 : if (op_type != binary_op)
5089 31196 : vec_oprnd1 = NULL;
5090 32308 : new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
5091 32308 : new_temp = make_ssa_name (vec_dest, new_stmt);
5092 32308 : gimple_set_lhs (new_stmt, new_temp);
5093 32308 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5094 :
5095 32308 : return new_stmt;
5096 : }
5097 :
5098 :
5099 : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5100 : For multi-step conversions store the resulting vectors and call the function
5101 : recursively. When NARROW_SRC_P is true, there's still a conversion after
5102 : narrowing, don't store the vectors in the SLP_NODE or in vector info of
5103 : the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5104 :
5105 : static void
5106 12138 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
5107 : int multi_step_cvt,
5108 : stmt_vec_info stmt_info,
5109 : vec<tree> &vec_dsts,
5110 : gimple_stmt_iterator *gsi,
5111 : slp_tree slp_node, code_helper code,
5112 : bool narrow_src_p)
5113 : {
5114 12138 : unsigned int i;
5115 12138 : tree vop0, vop1, new_tmp, vec_dest;
5116 :
5117 12138 : vec_dest = vec_dsts.pop ();
5118 :
5119 28734 : for (i = 0; i < vec_oprnds->length (); i += 2)
5120 : {
5121 : /* Create demotion operation. */
5122 16596 : vop0 = (*vec_oprnds)[i];
5123 16596 : vop1 = (*vec_oprnds)[i + 1];
5124 16596 : gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
5125 16596 : new_tmp = make_ssa_name (vec_dest, new_stmt);
5126 16596 : gimple_set_lhs (new_stmt, new_tmp);
5127 16596 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5128 16596 : if (multi_step_cvt || narrow_src_p)
5129 : /* Store the resulting vector for next recursive call,
5130 : or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5131 6798 : (*vec_oprnds)[i/2] = new_tmp;
5132 : else
5133 : {
5134 : /* This is the last step of the conversion sequence. Store the
5135 : vectors in SLP_NODE. */
5136 9798 : slp_node->push_vec_def (new_stmt);
5137 : }
5138 : }
5139 :
5140 : /* For multi-step demotion operations we first generate demotion operations
5141 : from the source type to the intermediate types, and then combine the
5142 : results (stored in VEC_OPRNDS) in demotion operation to the destination
5143 : type. */
5144 12138 : if (multi_step_cvt)
5145 : {
5146 : /* At each level of recursion we have half of the operands we had at the
5147 : previous level. */
5148 3015 : vec_oprnds->truncate ((i+1)/2);
5149 3015 : vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
5150 : multi_step_cvt - 1,
5151 : stmt_info, vec_dsts, gsi,
5152 3015 : slp_node, VEC_PACK_TRUNC_EXPR,
5153 : narrow_src_p);
5154 : }
5155 :
5156 12138 : vec_dsts.quick_push (vec_dest);
5157 12138 : }
5158 :
5159 :
5160 : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5161 : and VEC_OPRNDS1, for a binary operation associated with scalar statement
5162 : STMT_INFO. For multi-step conversions store the resulting vectors and
5163 : call the function recursively. When HALF is true only generate half
5164 : of the result. */
5165 :
5166 : static void
5167 11786 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
5168 : vec<tree> *vec_oprnds0,
5169 : vec<tree> *vec_oprnds1,
5170 : stmt_vec_info stmt_info, tree vec_dest,
5171 : gimple_stmt_iterator *gsi,
5172 : code_helper ch1,
5173 : code_helper ch2, int op_type,
5174 : bool half)
5175 : {
5176 11786 : int i;
5177 11786 : tree vop0, vop1, new_tmp1, new_tmp2;
5178 11786 : gimple *new_stmt1, *new_stmt2;
5179 11786 : vec<tree> vec_tmp = vNULL;
5180 :
5181 11786 : vec_tmp.create ((half ? 1 : 2) * vec_oprnds0->length ());
5182 27947 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5183 : {
5184 16161 : if (op_type == binary_op)
5185 556 : vop1 = (*vec_oprnds1)[i];
5186 : else
5187 : vop1 = NULL_TREE;
5188 :
5189 : /* Generate the two halves of promotion operation. */
5190 16161 : new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
5191 : op_type, vec_dest, gsi,
5192 : stmt_info);
5193 16161 : new_tmp1 = gimple_get_lhs (new_stmt1);
5194 16161 : vec_tmp.quick_push (new_tmp1);
5195 :
5196 16161 : if (vec_tmp.space (1))
5197 : {
5198 16147 : new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
5199 : op_type, vec_dest, gsi,
5200 : stmt_info);
5201 16147 : new_tmp2 = gimple_get_lhs (new_stmt2);
5202 16147 : vec_tmp.quick_push (new_tmp2);
5203 : }
5204 :
5205 16161 : if (!vec_tmp.space (1))
5206 : break;
5207 : }
5208 :
5209 11786 : vec_oprnds0->release ();
5210 11786 : *vec_oprnds0 = vec_tmp;
5211 11786 : }
5212 :
5213 : /* Create vectorized promotion stmts for widening stmts using only half the
5214 : potential vector size for input. */
5215 : static void
5216 60 : vect_create_half_widening_stmts (vec_info *vinfo,
5217 : vec<tree> *vec_oprnds0,
5218 : vec<tree> *vec_oprnds1,
5219 : stmt_vec_info stmt_info, tree vec_dest,
5220 : gimple_stmt_iterator *gsi,
5221 : code_helper code1,
5222 : int op_type)
5223 : {
5224 60 : int i;
5225 60 : tree vop0, vop1;
5226 60 : gimple *new_stmt1;
5227 60 : gimple *new_stmt2;
5228 60 : gimple *new_stmt3;
5229 60 : vec<tree> vec_tmp = vNULL;
5230 :
5231 60 : vec_tmp.create (vec_oprnds0->length ());
5232 180 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5233 : {
5234 60 : tree new_tmp1, new_tmp2, new_tmp3, out_type;
5235 :
5236 60 : gcc_assert (op_type == binary_op);
5237 60 : vop1 = (*vec_oprnds1)[i];
5238 :
5239 : /* Widen the first vector input. */
5240 60 : out_type = TREE_TYPE (vec_dest);
5241 60 : new_tmp1 = make_ssa_name (out_type);
5242 60 : new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
5243 60 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
5244 60 : if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
5245 : {
5246 : /* Widen the second vector input. */
5247 60 : new_tmp2 = make_ssa_name (out_type);
5248 60 : new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
5249 60 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
5250 : /* Perform the operation. With both vector inputs widened. */
5251 60 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
5252 : }
5253 : else
5254 : {
5255 : /* Perform the operation. With the single vector input widened. */
5256 0 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
5257 : }
5258 :
5259 60 : new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
5260 60 : gimple_assign_set_lhs (new_stmt3, new_tmp3);
5261 60 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
5262 :
5263 : /* Store the results for the next step. */
5264 60 : vec_tmp.quick_push (new_tmp3);
5265 : }
5266 :
5267 60 : vec_oprnds0->release ();
5268 60 : *vec_oprnds0 = vec_tmp;
5269 60 : }
5270 :
5271 :
5272 : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5273 : If COST_VEC is passed, calculate costs but don't change anything,
5274 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5275 : it, and insert it at GSI.
5276 : Return true if STMT_INFO is vectorizable in this way. */
5277 :
5278 : static bool
5279 2653985 : vectorizable_conversion (vec_info *vinfo,
5280 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5281 : slp_tree slp_node,
5282 : stmt_vector_for_cost *cost_vec)
5283 : {
5284 2653985 : tree vec_dest, cvt_op = NULL_TREE;
5285 2653985 : tree scalar_dest;
5286 2653985 : tree_code tc1;
5287 2653985 : code_helper code, code1, code2;
5288 2653985 : code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
5289 2653985 : tree new_temp;
5290 2653985 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5291 2653985 : poly_uint64 nunits_in;
5292 2653985 : poly_uint64 nunits_out;
5293 2653985 : tree vectype_out, vectype_in;
5294 2653985 : int i;
5295 2653985 : tree lhs_type, rhs_type;
5296 : /* For conversions between floating point and integer, there're 2 NARROW
5297 : cases. NARROW_SRC is for FLOAT_EXPR, means
5298 : integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5299 : This is safe when the range of the source integer can fit into the lower
5300 : precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5301 : floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5302 : For other conversions, when there's narrowing, NARROW_DST is used as
5303 : default. */
5304 2653985 : enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
5305 2653985 : vec<tree> vec_oprnds0 = vNULL;
5306 2653985 : vec<tree> vec_oprnds1 = vNULL;
5307 2653985 : tree vop0;
5308 2653985 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5309 2653985 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5310 2653985 : int multi_step_cvt = 0;
5311 2653985 : vec<tree> interm_types = vNULL;
5312 2653985 : tree intermediate_type, cvt_type = NULL_TREE;
5313 2653985 : int op_type;
5314 2653985 : unsigned short fltsz;
5315 :
5316 : /* Is STMT a vectorizable conversion? */
5317 :
5318 2653985 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5319 : return false;
5320 :
5321 2653985 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5322 244447 : && cost_vec)
5323 : return false;
5324 :
5325 2409538 : gimple* stmt = stmt_info->stmt;
5326 2409538 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
5327 : return false;
5328 :
5329 2347972 : if (gimple_get_lhs (stmt) == NULL_TREE
5330 2347972 : || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5331 : return false;
5332 :
5333 1516530 : if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5334 : return false;
5335 :
5336 1516530 : if (is_gimple_assign (stmt))
5337 : {
5338 1503089 : code = gimple_assign_rhs_code (stmt);
5339 1503089 : op_type = TREE_CODE_LENGTH ((tree_code) code);
5340 : }
5341 13441 : else if (gimple_call_internal_p (stmt))
5342 : {
5343 7950 : code = gimple_call_internal_fn (stmt);
5344 7950 : op_type = gimple_call_num_args (stmt);
5345 : }
5346 : else
5347 : return false;
5348 :
5349 1511039 : bool widen_arith = (code == WIDEN_MULT_EXPR
5350 1508408 : || code == WIDEN_LSHIFT_EXPR
5351 3019447 : || widening_fn_p (code));
5352 :
5353 1508408 : if (!widen_arith
5354 1508408 : && !CONVERT_EXPR_CODE_P (code)
5355 1335011 : && code != FIX_TRUNC_EXPR
5356 1333174 : && code != FLOAT_EXPR)
5357 : return false;
5358 :
5359 : /* Check types of lhs and rhs. */
5360 199111 : scalar_dest = gimple_get_lhs (stmt);
5361 199111 : lhs_type = TREE_TYPE (scalar_dest);
5362 199111 : vectype_out = SLP_TREE_VECTYPE (slp_node);
5363 :
5364 : /* Check the operands of the operation. */
5365 199111 : slp_tree slp_op0, slp_op1 = NULL;
5366 199111 : if (!vect_is_simple_use (vinfo, slp_node,
5367 : 0, &slp_op0, &dt[0], &vectype_in))
5368 : {
5369 0 : if (dump_enabled_p ())
5370 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5371 : "use not simple.\n");
5372 : return false;
5373 : }
5374 :
5375 199111 : rhs_type = TREE_TYPE (gimple_arg (stmt, 0));
5376 197274 : if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
5377 378960 : && !((INTEGRAL_TYPE_P (lhs_type)
5378 165068 : && INTEGRAL_TYPE_P (rhs_type))
5379 : || (SCALAR_FLOAT_TYPE_P (lhs_type)
5380 9830 : && SCALAR_FLOAT_TYPE_P (rhs_type))))
5381 : return false;
5382 :
5383 194160 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5384 173984 : && INTEGRAL_TYPE_P (lhs_type)
5385 337068 : && !type_has_mode_precision_p (lhs_type))
5386 : {
5387 529 : if (dump_enabled_p ())
5388 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5389 : "type conversion to bit-precision unsupported\n");
5390 : return false;
5391 : }
5392 :
5393 193631 : if (op_type == binary_op)
5394 : {
5395 2631 : gcc_assert (code == WIDEN_MULT_EXPR
5396 : || code == WIDEN_LSHIFT_EXPR
5397 : || widening_fn_p (code));
5398 :
5399 2631 : tree vectype1_in;
5400 2631 : if (!vect_is_simple_use (vinfo, slp_node, 1,
5401 : &slp_op1, &dt[1], &vectype1_in))
5402 : {
5403 0 : if (dump_enabled_p ())
5404 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5405 : "use not simple.\n");
5406 0 : return false;
5407 : }
5408 : /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5409 : OP1. */
5410 2631 : if (!vectype_in)
5411 176 : vectype_in = vectype1_in;
5412 : }
5413 :
5414 : /* If slp_op0 is an external or constant def, infer the vector type
5415 : from the scalar type. */
5416 193631 : if (!cost_vec)
5417 23573 : gcc_assert (vectype_in);
5418 193631 : if (!vectype_in)
5419 25107 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
5420 193631 : if (!vectype_in)
5421 : {
5422 260 : if (dump_enabled_p ())
5423 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5424 : "no vectype for scalar type %T\n", rhs_type);
5425 :
5426 : return false;
5427 : }
5428 :
5429 386742 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
5430 193371 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
5431 : {
5432 304 : if (dump_enabled_p ())
5433 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5434 : "can't convert between boolean and non "
5435 : "boolean vectors %T\n", rhs_type);
5436 :
5437 : return false;
5438 : }
5439 :
5440 193067 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
5441 193067 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5442 193067 : if (known_eq (nunits_out, nunits_in))
5443 92021 : if (widen_arith)
5444 : modifier = WIDEN;
5445 : else
5446 193067 : modifier = NONE;
5447 101046 : else if (multiple_p (nunits_out, nunits_in))
5448 : modifier = NARROW_DST;
5449 : else
5450 : {
5451 57191 : gcc_checking_assert (multiple_p (nunits_in, nunits_out));
5452 : modifier = WIDEN;
5453 : }
5454 :
5455 193067 : bool found_mode = false;
5456 193067 : scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
5457 193067 : scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
5458 193067 : opt_scalar_mode rhs_mode_iter;
5459 193067 : auto_vec<std::pair<tree, tree_code>, 2> converts;
5460 193067 : bool evenodd_ok = false;
5461 :
5462 : /* Supportable by target? */
5463 193067 : switch (modifier)
5464 : {
5465 91610 : case NONE:
5466 91610 : if (code != FIX_TRUNC_EXPR
5467 90489 : && code != FLOAT_EXPR
5468 170809 : && !CONVERT_EXPR_CODE_P (code))
5469 : return false;
5470 91610 : gcc_assert (code.is_tree_code ());
5471 91610 : if (supportable_indirect_convert_operation (code,
5472 : vectype_out, vectype_in,
5473 : converts, slp_op0))
5474 : {
5475 84921 : gcc_assert (converts.length () <= 2);
5476 84921 : if (converts.length () == 1)
5477 : {
5478 84847 : code1 = converts[0].second;
5479 14773 : if (CONVERT_EXPR_CODE_P (code)
5480 159649 : && tree_nop_conversion_p (TREE_TYPE (vectype_out),
5481 74802 : TREE_TYPE (vectype_in)))
5482 : /* NOP conversions are handled by vectorizable_assignment. */
5483 : return false;
5484 : }
5485 : else
5486 : {
5487 74 : cvt_type = NULL_TREE;
5488 74 : multi_step_cvt = converts.length () - 1;
5489 74 : codecvt1 = converts[0].second;
5490 74 : code1 = converts[1].second;
5491 74 : interm_types.safe_push (converts[0].first);
5492 : }
5493 : break;
5494 : }
5495 :
5496 : /* FALLTHRU */
5497 6689 : unsupported:
5498 19582 : if (dump_enabled_p ())
5499 560 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5500 : "conversion not supported by target.\n");
5501 : return false;
5502 :
5503 57602 : case WIDEN:
5504 57602 : if (known_eq (nunits_in, nunits_out))
5505 : {
5506 822 : if (!(code.is_tree_code ()
5507 411 : && supportable_half_widening_operation ((tree_code) code,
5508 : vectype_out, vectype_in,
5509 : &tc1)))
5510 129 : goto unsupported;
5511 282 : code1 = tc1;
5512 282 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5513 : break;
5514 : }
5515 : /* Elements in a vector can only be reordered if used in a reduction
5516 : operation only. */
5517 57191 : if (code == WIDEN_MULT_EXPR
5518 2220 : && loop_vinfo
5519 2079 : && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
5520 : /* For a SLP reduction we cannot swizzle lanes, detecting a
5521 : reduction chain isn't possible here. */
5522 59248 : && SLP_TREE_LANES (slp_node) == 1)
5523 : {
5524 : /* ??? There is no way to look for SLP uses, so work on
5525 : the stmt and what the stmt-based cycle detection gives us. */
5526 1955 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5527 1955 : stmt_vec_info use_stmt_info
5528 1955 : = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
5529 1955 : if (use_stmt_info
5530 1806 : && STMT_VINFO_REDUC_DEF (use_stmt_info))
5531 57191 : evenodd_ok = true;
5532 : }
5533 57191 : if (supportable_widening_operation (code, vectype_out, vectype_in,
5534 : evenodd_ok, &code1,
5535 : &code2, &multi_step_cvt,
5536 : &interm_types))
5537 : {
5538 : /* Binary widening operation can only be supported directly by the
5539 : architecture. */
5540 50762 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5541 : break;
5542 : }
5543 :
5544 6429 : if (code != FLOAT_EXPR
5545 6795 : || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
5546 6246 : goto unsupported;
5547 :
5548 183 : fltsz = GET_MODE_SIZE (lhs_mode);
5549 270 : FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
5550 : {
5551 270 : rhs_mode = rhs_mode_iter.require ();
5552 540 : if (GET_MODE_SIZE (rhs_mode) > fltsz)
5553 : break;
5554 :
5555 270 : cvt_type
5556 270 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5557 270 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5558 270 : if (cvt_type == NULL_TREE)
5559 0 : goto unsupported;
5560 :
5561 540 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5562 : {
5563 78 : gcc_assert (code.is_tree_code ());
5564 78 : if (!supportable_convert_operation ((tree_code) code, vectype_out,
5565 : cvt_type))
5566 22 : goto unsupported;
5567 56 : codecvt1 = code;
5568 : }
5569 192 : else if (!supportable_widening_operation (code, vectype_out,
5570 : cvt_type, evenodd_ok,
5571 : &codecvt1,
5572 : &codecvt2, &multi_step_cvt,
5573 : &interm_types))
5574 87 : continue;
5575 : else
5576 105 : gcc_assert (multi_step_cvt == 0);
5577 :
5578 161 : if (supportable_widening_operation (NOP_EXPR, cvt_type,
5579 : vectype_in, evenodd_ok, &code1,
5580 : &code2, &multi_step_cvt,
5581 : &interm_types))
5582 : {
5583 : found_mode = true;
5584 : break;
5585 : }
5586 : }
5587 :
5588 161 : if (!found_mode)
5589 0 : goto unsupported;
5590 :
5591 322 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5592 56 : codecvt2 = ERROR_MARK;
5593 : else
5594 : {
5595 105 : multi_step_cvt++;
5596 105 : interm_types.safe_push (cvt_type);
5597 105 : cvt_type = NULL_TREE;
5598 : }
5599 : break;
5600 :
5601 43855 : case NARROW_DST:
5602 43855 : gcc_assert (op_type == unary_op);
5603 43855 : if (supportable_narrowing_operation (code, vectype_out, vectype_in,
5604 : &code1, &multi_step_cvt,
5605 : &interm_types))
5606 : break;
5607 :
5608 19890 : if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
5609 986 : goto unsupported;
5610 :
5611 5644 : if (code == FIX_TRUNC_EXPR)
5612 : {
5613 107 : cvt_type
5614 107 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5615 107 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5616 107 : if (cvt_type == NULL_TREE)
5617 0 : goto unsupported;
5618 107 : if (supportable_convert_operation ((tree_code) code, cvt_type,
5619 : vectype_in))
5620 105 : codecvt1 = code;
5621 : else
5622 2 : goto unsupported;
5623 105 : if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
5624 : &code1, &multi_step_cvt,
5625 : &interm_types))
5626 : break;
5627 : }
5628 : /* If slp_op0 can be represented with low precision integer,
5629 : truncate it to cvt_type and the do FLOAT_EXPR. */
5630 5537 : else if (code == FLOAT_EXPR)
5631 : {
5632 792 : if (cost_vec)
5633 : {
5634 784 : wide_int op_min_value, op_max_value;
5635 784 : tree def;
5636 :
5637 : /* ??? Merge ranges in case of more than one lane. */
5638 784 : if (SLP_TREE_LANES (slp_op0) != 1
5639 133 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
5640 917 : || !vect_get_range_info (def, &op_min_value, &op_max_value))
5641 755 : goto unsupported;
5642 :
5643 29 : if ((wi::min_precision (op_max_value, SIGNED)
5644 29 : > GET_MODE_BITSIZE (lhs_mode))
5645 29 : || (wi::min_precision (op_min_value, SIGNED)
5646 27 : > GET_MODE_BITSIZE (lhs_mode)))
5647 2 : goto unsupported;
5648 784 : }
5649 :
5650 35 : cvt_type
5651 35 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
5652 35 : cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
5653 35 : if (cvt_type == NULL_TREE)
5654 0 : goto unsupported;
5655 35 : if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
5656 : &code1, &multi_step_cvt,
5657 : &interm_types))
5658 2 : goto unsupported;
5659 33 : if (supportable_convert_operation ((tree_code) code, vectype_out,
5660 : cvt_type))
5661 : {
5662 33 : codecvt1 = code;
5663 33 : modifier = NARROW_SRC;
5664 33 : break;
5665 : }
5666 : }
5667 :
5668 4749 : goto unsupported;
5669 :
5670 : default:
5671 : gcc_unreachable ();
5672 : }
5673 :
5674 109751 : if (modifier == WIDEN
5675 109751 : && loop_vinfo
5676 49544 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
5677 130818 : && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
5678 21045 : || widening_evenodd_fn_p (code1)))
5679 : {
5680 22 : if (dump_enabled_p ())
5681 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5682 : "can't use a fully-masked loop because"
5683 : " widening operation on even/odd elements"
5684 : " mixes up lanes.\n");
5685 22 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
5686 : }
5687 :
5688 109751 : if (cost_vec) /* transformation not required. */
5689 : {
5690 86178 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
5691 86178 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
5692 : {
5693 0 : if (dump_enabled_p ())
5694 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5695 : "incompatible vector types for invariants\n");
5696 : return false;
5697 : }
5698 86178 : DUMP_VECT_SCOPE ("vectorizable_conversion");
5699 86178 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
5700 86178 : if (modifier == NONE)
5701 : {
5702 16746 : SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
5703 16746 : vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
5704 : slp_node, cost_vec);
5705 : }
5706 69432 : else if (modifier == NARROW_SRC || modifier == NARROW_DST)
5707 : {
5708 28236 : SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
5709 : /* The final packing step produces one vector result per copy. */
5710 28236 : vect_model_promotion_demotion_cost (slp_node, nvectors,
5711 : multi_step_cvt, cost_vec,
5712 : widen_arith);
5713 : }
5714 : else
5715 : {
5716 41196 : SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
5717 : /* The initial unpacking step produces two vector results
5718 : per copy. MULTI_STEP_CVT is 0 for a single conversion,
5719 : so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5720 41196 : vect_model_promotion_demotion_cost (slp_node,
5721 : nvectors >> multi_step_cvt,
5722 : multi_step_cvt, cost_vec,
5723 : widen_arith);
5724 : }
5725 86178 : interm_types.release ();
5726 86178 : return true;
5727 86178 : }
5728 :
5729 : /* Transform. */
5730 23573 : if (dump_enabled_p ())
5731 4294 : dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
5732 :
5733 : /* In case of multi-step conversion, we first generate conversion operations
5734 : to the intermediate types, and then from that types to the final one.
5735 : We create vector destinations for the intermediate type (TYPES) received
5736 : from supportable_*_operation, and store them in the correct order
5737 : for future use in vect_create_vectorized_*_stmts (). */
5738 23573 : auto_vec<tree> vec_dsts (multi_step_cvt + 1);
5739 23573 : bool widen_or_narrow_float_p
5740 23573 : = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
5741 23573 : vec_dest = vect_create_destination_var (scalar_dest,
5742 : widen_or_narrow_float_p
5743 : ? cvt_type : vectype_out);
5744 23573 : vec_dsts.quick_push (vec_dest);
5745 :
5746 23573 : if (multi_step_cvt)
5747 : {
5748 9258 : for (i = interm_types.length () - 1;
5749 9258 : interm_types.iterate (i, &intermediate_type); i--)
5750 : {
5751 4873 : vec_dest = vect_create_destination_var (scalar_dest,
5752 : intermediate_type);
5753 4873 : vec_dsts.quick_push (vec_dest);
5754 : }
5755 : }
5756 :
5757 23573 : if (cvt_type)
5758 76 : vec_dest = vect_create_destination_var (scalar_dest,
5759 : widen_or_narrow_float_p
5760 : ? vectype_out : cvt_type);
5761 :
5762 23573 : unsigned num_vectors = vect_get_num_copies (vinfo, slp_node);
5763 23573 : switch (modifier)
5764 : {
5765 4441 : case NONE:
5766 4441 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5767 : /* vec_dest is intermediate type operand when multi_step_cvt. */
5768 4441 : if (multi_step_cvt)
5769 : {
5770 21 : cvt_op = vec_dest;
5771 21 : vec_dest = vec_dsts[0];
5772 : }
5773 :
5774 9305 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5775 : {
5776 : /* Arguments are ready, create the new vector stmt. */
5777 4864 : gimple* new_stmt;
5778 4864 : if (multi_step_cvt)
5779 : {
5780 21 : gcc_assert (multi_step_cvt == 1);
5781 21 : new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
5782 21 : new_temp = make_ssa_name (cvt_op, new_stmt);
5783 21 : gimple_assign_set_lhs (new_stmt, new_temp);
5784 21 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5785 21 : vop0 = new_temp;
5786 : }
5787 4864 : if (code1 == COND_EXPR)
5788 : {
5789 1 : gcc_assert (!multi_step_cvt);
5790 1 : new_stmt
5791 2 : = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
5792 : build_minus_one_cst
5793 1 : (TREE_TYPE (vec_dest)),
5794 1 : build_zero_cst (TREE_TYPE (vec_dest)));
5795 1 : new_temp = make_ssa_name (vec_dest, new_stmt);
5796 1 : gimple_set_lhs (new_stmt, new_temp);
5797 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5798 1 : tree new_temp2 = make_ssa_name (vectype_out);
5799 1 : new_stmt = gimple_build_assign (new_temp2,
5800 : build1 (VIEW_CONVERT_EXPR,
5801 : vectype_out, new_temp));
5802 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5803 1 : slp_node->push_vec_def (new_temp2);
5804 : }
5805 : else
5806 : {
5807 4863 : new_stmt = vect_gimple_build (vec_dest, code1, vop0);
5808 4863 : new_temp = make_ssa_name (vec_dest, new_stmt);
5809 4863 : gimple_set_lhs (new_stmt, new_temp);
5810 4863 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5811 :
5812 4863 : slp_node->push_vec_def (new_stmt);
5813 : }
5814 : }
5815 : break;
5816 :
5817 10009 : case WIDEN:
5818 : /* In case the vectorization factor (VF) is bigger than the number
5819 : of elements that we can fit in a vectype (nunits), we have to
5820 : generate more than one vector stmt - i.e - we need to "unroll"
5821 : the vector stmt by a factor VF/nunits. */
5822 10009 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
5823 10009 : code != WIDEN_LSHIFT_EXPR && slp_op1, &vec_oprnds1);
5824 10009 : if (code == WIDEN_LSHIFT_EXPR)
5825 : {
5826 0 : int oprnds_size = vec_oprnds0.length ();
5827 0 : vec_oprnds1.create (oprnds_size);
5828 0 : for (i = 0; i < oprnds_size; ++i)
5829 0 : vec_oprnds1.quick_push (vect_get_slp_scalar_def (slp_op1, 0));
5830 : }
5831 : /* Arguments are ready. Create the new vector stmts. */
5832 21855 : for (i = multi_step_cvt; i >= 0; i--)
5833 : {
5834 11846 : tree this_dest = vec_dsts[i];
5835 11846 : code_helper c1 = code1, c2 = code2;
5836 11846 : if (i == 0 && codecvt2 != ERROR_MARK)
5837 : {
5838 48 : c1 = codecvt1;
5839 48 : c2 = codecvt2;
5840 : }
5841 11846 : if (known_eq (nunits_out, nunits_in))
5842 60 : vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
5843 : stmt_info, this_dest, gsi, c1,
5844 : op_type);
5845 : else
5846 : /* ??? For constant/external inputs we can end up with
5847 : excess lanes. When the number of inputs already match
5848 : the number of required outputs request half of the
5849 : lanes (gcc.dg/vect/O3-vect-pr32243.c). Low coverage
5850 : makes this likely incomplete. */
5851 11786 : vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
5852 : &vec_oprnds1, stmt_info,
5853 : this_dest, gsi,
5854 : c1, c2, op_type,
5855 11786 : vec_oprnds0.length ()
5856 : == num_vectors);
5857 : }
5858 :
5859 38102 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5860 : {
5861 28093 : gimple *new_stmt;
5862 28093 : if (cvt_type)
5863 : {
5864 120 : new_temp = make_ssa_name (vec_dest);
5865 120 : new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5866 120 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5867 : }
5868 : else
5869 27973 : new_stmt = SSA_NAME_DEF_STMT (vop0);
5870 :
5871 28093 : slp_node->push_vec_def (new_stmt);
5872 : }
5873 : break;
5874 :
5875 9123 : case NARROW_SRC:
5876 9123 : case NARROW_DST:
5877 : /* In case the vectorization factor (VF) is bigger than the number
5878 : of elements that we can fit in a vectype (nunits), we have to
5879 : generate more than one vector stmt - i.e - we need to "unroll"
5880 : the vector stmt by a factor VF/nunits. */
5881 9123 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5882 : /* Arguments are ready. Create the new vector stmts. */
5883 9123 : if (cvt_type && modifier == NARROW_DST)
5884 153 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5885 : {
5886 124 : new_temp = make_ssa_name (vec_dest);
5887 124 : gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5888 124 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5889 124 : vec_oprnds0[i] = new_temp;
5890 : }
5891 :
5892 9123 : vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
5893 : multi_step_cvt,
5894 : stmt_info, vec_dsts, gsi,
5895 : slp_node, code1,
5896 : modifier == NARROW_SRC);
5897 : /* After demoting slp_op0 to cvt_type, convert it to dest. */
5898 9123 : if (cvt_type && code == FLOAT_EXPR)
5899 : {
5900 16 : for (unsigned int i = 0; i != vec_oprnds0.length() / 2; i++)
5901 : {
5902 : /* Arguments are ready, create the new vector stmt. */
5903 8 : gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
5904 8 : gimple *new_stmt
5905 8 : = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
5906 8 : new_temp = make_ssa_name (vec_dest, new_stmt);
5907 8 : gimple_set_lhs (new_stmt, new_temp);
5908 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5909 :
5910 : /* This is the last step of the conversion sequence. Store the
5911 : vectors in SLP_NODE or in vector info of the scalar statement
5912 : (or in STMT_VINFO_RELATED_STMT chain). */
5913 8 : slp_node->push_vec_def (new_stmt);
5914 : }
5915 : }
5916 : break;
5917 : }
5918 :
5919 23573 : vec_oprnds0.release ();
5920 23573 : vec_oprnds1.release ();
5921 23573 : interm_types.release ();
5922 :
5923 23573 : return true;
5924 193067 : }
5925 :
5926 : /* Return true if we can assume from the scalar form of STMT_INFO that
5927 : neither the scalar nor the vector forms will generate code. STMT_INFO
5928 : is known not to involve a data reference. */
5929 :
5930 : bool
5931 3292856 : vect_nop_conversion_p (stmt_vec_info stmt_info)
5932 : {
5933 3292856 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5934 2994340 : if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
5935 : return false;
5936 :
5937 984621 : tree lhs = gimple_assign_lhs (stmt);
5938 984621 : tree_code code = gimple_assign_rhs_code (stmt);
5939 984621 : tree rhs = gimple_assign_rhs1 (stmt);
5940 :
5941 984621 : if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
5942 : return true;
5943 :
5944 981375 : if (CONVERT_EXPR_CODE_P (code))
5945 245753 : return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
5946 :
5947 : return false;
5948 : }
5949 :
5950 : /* Function vectorizable_assignment.
5951 :
5952 : Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5953 : If COST_VEC is passed, calculate costs but don't change anything,
5954 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5955 : it, and insert it at GSI.
5956 : Return true if STMT_INFO is vectorizable in this way. */
5957 :
5958 : static bool
5959 2136864 : vectorizable_assignment (vec_info *vinfo,
5960 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5961 : slp_tree slp_node,
5962 : stmt_vector_for_cost *cost_vec)
5963 : {
5964 2136864 : tree vec_dest;
5965 2136864 : tree scalar_dest;
5966 2136864 : tree op;
5967 2136864 : tree new_temp;
5968 2136864 : enum vect_def_type dt[1] = {vect_unknown_def_type};
5969 2136864 : int i;
5970 2136864 : vec<tree> vec_oprnds = vNULL;
5971 2136864 : tree vop;
5972 2136864 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5973 2136864 : enum tree_code code;
5974 2136864 : tree vectype_in;
5975 :
5976 2136864 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5977 : return false;
5978 :
5979 2136864 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5980 244447 : && cost_vec)
5981 : return false;
5982 :
5983 : /* Is vectorizable assignment? */
5984 1892417 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5985 1816080 : if (!stmt)
5986 : return false;
5987 :
5988 1816080 : scalar_dest = gimple_assign_lhs (stmt);
5989 1816080 : if (TREE_CODE (scalar_dest) != SSA_NAME)
5990 : return false;
5991 :
5992 985968 : if (STMT_VINFO_DATA_REF (stmt_info))
5993 : return false;
5994 :
5995 416867 : code = gimple_assign_rhs_code (stmt);
5996 416867 : if (!(gimple_assign_single_p (stmt)
5997 415241 : || code == PAREN_EXPR
5998 413991 : || CONVERT_EXPR_CODE_P (code)))
5999 : return false;
6000 :
6001 104624 : tree vectype = SLP_TREE_VECTYPE (slp_node);
6002 104624 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6003 :
6004 104624 : slp_tree slp_op;
6005 104624 : if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
6006 : &dt[0], &vectype_in))
6007 : {
6008 0 : if (dump_enabled_p ())
6009 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6010 : "use not simple.\n");
6011 : return false;
6012 : }
6013 104624 : if (!vectype_in)
6014 20094 : vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
6015 :
6016 : /* We can handle VIEW_CONVERT conversions that do not change the number
6017 : of elements or the vector size or other conversions when the component
6018 : types are nop-convertible. */
6019 104624 : if (!vectype_in
6020 109189 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
6021 91858 : || (code == VIEW_CONVERT_EXPR
6022 2996 : && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
6023 2996 : GET_MODE_SIZE (TYPE_MODE (vectype_in))))
6024 196482 : || (CONVERT_EXPR_CODE_P (code)
6025 89014 : && !tree_nop_conversion_p (TREE_TYPE (vectype),
6026 89014 : TREE_TYPE (vectype_in))))
6027 : return false;
6028 :
6029 261406 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
6030 : {
6031 27 : if (dump_enabled_p ())
6032 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6033 : "can't convert between boolean and non "
6034 0 : "boolean vectors %T\n", TREE_TYPE (op));
6035 :
6036 : return false;
6037 : }
6038 :
6039 : /* We do not handle bit-precision changes. */
6040 87266 : if ((CONVERT_EXPR_CODE_P (code)
6041 2844 : || code == VIEW_CONVERT_EXPR)
6042 85920 : && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6043 84565 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6044 85413 : || (INTEGRAL_TYPE_P (TREE_TYPE (op))
6045 77263 : && !type_has_mode_precision_p (TREE_TYPE (op))))
6046 : /* But a conversion that does not change the bit-pattern is ok. */
6047 88322 : && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6048 1056 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
6049 1056 : && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
6050 1056 : > TYPE_PRECISION (TREE_TYPE (op)))
6051 549 : && TYPE_UNSIGNED (TREE_TYPE (op)))
6052 569 : || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
6053 569 : == TYPE_PRECISION (TREE_TYPE (op))))))
6054 : {
6055 333 : if (dump_enabled_p ())
6056 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6057 : "type conversion to/from bit-precision "
6058 : "unsupported.\n");
6059 : return false;
6060 : }
6061 :
6062 86933 : if (cost_vec) /* transformation not required. */
6063 : {
6064 70567 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
6065 : {
6066 0 : if (dump_enabled_p ())
6067 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6068 : "incompatible vector types for invariants\n");
6069 : return false;
6070 : }
6071 70567 : SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
6072 70567 : DUMP_VECT_SCOPE ("vectorizable_assignment");
6073 70567 : if (!vect_nop_conversion_p (stmt_info))
6074 1047 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6075 70567 : return true;
6076 : }
6077 :
6078 : /* Transform. */
6079 16366 : if (dump_enabled_p ())
6080 3639 : dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
6081 :
6082 : /* Handle def. */
6083 16366 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6084 :
6085 : /* Handle use. */
6086 16366 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
6087 :
6088 : /* Arguments are ready. create the new vector stmt. */
6089 53220 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
6090 : {
6091 20488 : if (CONVERT_EXPR_CODE_P (code)
6092 722 : || code == VIEW_CONVERT_EXPR)
6093 19924 : vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
6094 20488 : gassign *new_stmt = gimple_build_assign (vec_dest, vop);
6095 20488 : new_temp = make_ssa_name (vec_dest, new_stmt);
6096 20488 : gimple_assign_set_lhs (new_stmt, new_temp);
6097 20488 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6098 20488 : slp_node->push_vec_def (new_stmt);
6099 : }
6100 :
6101 16366 : vec_oprnds.release ();
6102 16366 : return true;
6103 : }
6104 :
6105 :
6106 : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6107 : either as shift by a scalar or by a vector. */
6108 :
6109 : bool
6110 301226 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
6111 : {
6112 301226 : optab optab;
6113 301226 : tree vectype;
6114 :
6115 301226 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6116 301226 : if (!vectype)
6117 : return false;
6118 :
6119 301226 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6120 301226 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6121 : return true;
6122 :
6123 264366 : optab = optab_for_tree_code (code, vectype, optab_vector);
6124 264366 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6125 : return true;
6126 :
6127 : return false;
6128 : }
6129 :
6130 :
6131 : /* Function vectorizable_shift.
6132 :
6133 : Check if STMT_INFO performs a shift operation that can be vectorized.
6134 : If COST_VEC is passed, calculate costs but don't change anything,
6135 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6136 : it, and insert it at GSI.
6137 : Return true if STMT_INFO is vectorizable in this way. */
6138 :
6139 : static bool
6140 771251 : vectorizable_shift (vec_info *vinfo,
6141 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6142 : slp_tree slp_node,
6143 : stmt_vector_for_cost *cost_vec)
6144 : {
6145 771251 : tree vec_dest;
6146 771251 : tree scalar_dest;
6147 771251 : tree op0, op1 = NULL;
6148 771251 : tree vec_oprnd1 = NULL_TREE;
6149 771251 : tree vectype;
6150 771251 : enum tree_code code;
6151 771251 : machine_mode vec_mode;
6152 771251 : tree new_temp;
6153 771251 : optab optab;
6154 771251 : int icode;
6155 771251 : machine_mode optab_op2_mode;
6156 771251 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
6157 771251 : poly_uint64 nunits_in;
6158 771251 : poly_uint64 nunits_out;
6159 771251 : tree vectype_out;
6160 771251 : tree op1_vectype;
6161 771251 : int i;
6162 771251 : vec<tree> vec_oprnds0 = vNULL;
6163 771251 : vec<tree> vec_oprnds1 = vNULL;
6164 771251 : tree vop0, vop1;
6165 771251 : unsigned int k;
6166 771251 : bool scalar_shift_arg = true;
6167 771251 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6168 771251 : bool incompatible_op1_vectype_p = false;
6169 :
6170 771251 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6171 : return false;
6172 :
6173 771251 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6174 244447 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
6175 242957 : && cost_vec)
6176 : return false;
6177 :
6178 : /* Is STMT a vectorizable binary/unary operation? */
6179 528294 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6180 452936 : if (!stmt)
6181 : return false;
6182 :
6183 452936 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
6184 : return false;
6185 :
6186 452378 : code = gimple_assign_rhs_code (stmt);
6187 :
6188 452378 : if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
6189 : || code == RROTATE_EXPR))
6190 : return false;
6191 :
6192 59634 : scalar_dest = gimple_assign_lhs (stmt);
6193 59634 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6194 59634 : if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6195 : {
6196 0 : if (dump_enabled_p ())
6197 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6198 : "bit-precision shifts not supported.\n");
6199 : return false;
6200 : }
6201 :
6202 59634 : slp_tree slp_op0;
6203 59634 : if (!vect_is_simple_use (vinfo, slp_node,
6204 : 0, &op0, &slp_op0, &dt[0], &vectype))
6205 : {
6206 0 : if (dump_enabled_p ())
6207 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6208 : "use not simple.\n");
6209 : return false;
6210 : }
6211 : /* If op0 is an external or constant def, infer the vector type
6212 : from the scalar type. */
6213 59634 : if (!vectype)
6214 14885 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
6215 59634 : if (!cost_vec)
6216 8776 : gcc_assert (vectype);
6217 59634 : if (!vectype)
6218 : {
6219 0 : if (dump_enabled_p ())
6220 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6221 : "no vectype for scalar type\n");
6222 : return false;
6223 : }
6224 :
6225 59634 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6226 59634 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6227 59634 : if (maybe_ne (nunits_out, nunits_in))
6228 : return false;
6229 :
6230 59634 : stmt_vec_info op1_def_stmt_info;
6231 59634 : slp_tree slp_op1;
6232 59634 : if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
6233 : &dt[1], &op1_vectype, &op1_def_stmt_info))
6234 : {
6235 0 : if (dump_enabled_p ())
6236 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6237 : "use not simple.\n");
6238 : return false;
6239 : }
6240 :
6241 : /* Determine whether the shift amount is a vector, or scalar. If the
6242 : shift/rotate amount is a vector, use the vector/vector shift optabs. */
6243 :
6244 59634 : if ((dt[1] == vect_internal_def
6245 59634 : || dt[1] == vect_induction_def
6246 42965 : || dt[1] == vect_nested_cycle)
6247 16687 : && SLP_TREE_LANES (slp_node) == 1)
6248 : scalar_shift_arg = false;
6249 43002 : else if (dt[1] == vect_constant_def
6250 : || dt[1] == vect_external_def
6251 43002 : || dt[1] == vect_internal_def)
6252 : {
6253 : /* In SLP, need to check whether the shift count is the same,
6254 : in loops if it is a constant or invariant, it is always
6255 : a scalar shift. */
6256 42996 : vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
6257 42996 : stmt_vec_info slpstmt_info;
6258 :
6259 111674 : FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
6260 68678 : if (slpstmt_info)
6261 : {
6262 68678 : gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
6263 137356 : if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
6264 68678 : scalar_shift_arg = false;
6265 : }
6266 :
6267 : /* For internal SLP defs we have to make sure we see scalar stmts
6268 : for all vector elements.
6269 : ??? For different vectors we could resort to a different
6270 : scalar shift operand but code-generation below simply always
6271 : takes the first. */
6272 42996 : if (dt[1] == vect_internal_def
6273 43045 : && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
6274 49 : stmts.length ()))
6275 : scalar_shift_arg = false;
6276 :
6277 : /* If the shift amount is computed by a pattern stmt we cannot
6278 : use the scalar amount directly thus give up and use a vector
6279 : shift. */
6280 42996 : if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
6281 : scalar_shift_arg = false;
6282 : }
6283 : else
6284 : {
6285 6 : if (dump_enabled_p ())
6286 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6287 : "operand mode requires invariant argument.\n");
6288 : return false;
6289 : }
6290 :
6291 : /* Vector shifted by vector. */
6292 59666 : bool was_scalar_shift_arg = scalar_shift_arg;
6293 42987 : if (!scalar_shift_arg)
6294 : {
6295 16679 : optab = optab_for_tree_code (code, vectype, optab_vector);
6296 16679 : if (dump_enabled_p ())
6297 1207 : dump_printf_loc (MSG_NOTE, vect_location,
6298 : "vector/vector shift/rotate found.\n");
6299 :
6300 16679 : if (!op1_vectype)
6301 15 : op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
6302 : slp_op1);
6303 16679 : incompatible_op1_vectype_p
6304 33358 : = (op1_vectype == NULL_TREE
6305 16679 : || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
6306 16679 : TYPE_VECTOR_SUBPARTS (vectype))
6307 33356 : || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
6308 16679 : if (incompatible_op1_vectype_p
6309 7 : && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
6310 1 : || slp_op1->refcnt != 1))
6311 : {
6312 6 : if (dump_enabled_p ())
6313 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6314 : "unusable type for last operand in"
6315 : " vector/vector shift/rotate.\n");
6316 : return false;
6317 : }
6318 : }
6319 : /* See if the machine has a vector shifted by scalar insn and if not
6320 : then see if it has a vector shifted by vector insn. */
6321 : else
6322 : {
6323 42949 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6324 42949 : if (optab
6325 42949 : && can_implement_p (optab, TYPE_MODE (vectype)))
6326 : {
6327 42648 : if (dump_enabled_p ())
6328 4954 : dump_printf_loc (MSG_NOTE, vect_location,
6329 : "vector/scalar shift/rotate found.\n");
6330 : }
6331 : else
6332 : {
6333 301 : optab = optab_for_tree_code (code, vectype, optab_vector);
6334 301 : if (optab
6335 301 : && can_implement_p (optab, TYPE_MODE (vectype)))
6336 : {
6337 0 : scalar_shift_arg = false;
6338 :
6339 0 : if (dump_enabled_p ())
6340 0 : dump_printf_loc (MSG_NOTE, vect_location,
6341 : "vector/vector shift/rotate found.\n");
6342 :
6343 0 : if (!op1_vectype)
6344 0 : op1_vectype = get_vectype_for_scalar_type (vinfo,
6345 0 : TREE_TYPE (op1),
6346 : slp_op1);
6347 :
6348 : /* Unlike the other binary operators, shifts/rotates have
6349 : the rhs being int, instead of the same type as the lhs,
6350 : so make sure the scalar is the right type if we are
6351 : dealing with vectors of long long/long/short/char. */
6352 0 : incompatible_op1_vectype_p
6353 0 : = (!op1_vectype
6354 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype),
6355 0 : TREE_TYPE (op1)));
6356 : if (incompatible_op1_vectype_p
6357 0 : && dt[1] == vect_internal_def)
6358 : {
6359 0 : if (dump_enabled_p ())
6360 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6361 : "unusable type for last operand in"
6362 : " vector/vector shift/rotate.\n");
6363 : return false;
6364 : }
6365 : }
6366 : }
6367 : }
6368 :
6369 : /* Supportable by target? */
6370 59622 : if (!optab)
6371 : {
6372 0 : if (dump_enabled_p ())
6373 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6374 : "no shift optab for %s and %T.\n",
6375 : get_tree_code_name (code), vectype);
6376 : return false;
6377 : }
6378 59622 : vec_mode = TYPE_MODE (vectype);
6379 59622 : icode = (int) optab_handler (optab, vec_mode);
6380 59622 : if (icode == CODE_FOR_nothing)
6381 : {
6382 6587 : if (dump_enabled_p ())
6383 930 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6384 : "shift op not supported by target.\n");
6385 : return false;
6386 : }
6387 : /* vector lowering cannot optimize vector shifts using word arithmetic. */
6388 53035 : if (vect_emulated_vector_p (vectype))
6389 : return false;
6390 :
6391 53035 : if (cost_vec) /* transformation not required. */
6392 : {
6393 44259 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6394 44259 : || ((!scalar_shift_arg || dt[1] == vect_internal_def)
6395 8237 : && (!incompatible_op1_vectype_p
6396 1 : || dt[1] == vect_constant_def)
6397 8237 : && !vect_maybe_update_slp_op_vectype
6398 8237 : (slp_op1,
6399 : incompatible_op1_vectype_p ? vectype : op1_vectype)))
6400 : {
6401 0 : if (dump_enabled_p ())
6402 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6403 : "incompatible vector types for invariants\n");
6404 : return false;
6405 : }
6406 : /* Now adjust the constant shift amount in place. */
6407 44259 : if (incompatible_op1_vectype_p
6408 1 : && dt[1] == vect_constant_def)
6409 : {
6410 1 : unsigned group_size = SLP_TREE_LANES (slp_op1);
6411 2 : gcc_assert (SLP_TREE_SCALAR_OPS (slp_op1).length () == group_size);
6412 5 : for (unsigned i = 0; i < group_size; ++i)
6413 : {
6414 4 : SLP_TREE_SCALAR_OPS (slp_op1)[i]
6415 4 : = fold_convert (TREE_TYPE (vectype),
6416 : SLP_TREE_SCALAR_OPS (slp_op1)[i]);
6417 4 : gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
6418 : == INTEGER_CST));
6419 : }
6420 : }
6421 44259 : SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
6422 44259 : DUMP_VECT_SCOPE ("vectorizable_shift");
6423 44259 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6424 44259 : return true;
6425 : }
6426 :
6427 : /* Transform. */
6428 :
6429 8776 : if (dump_enabled_p ())
6430 2035 : dump_printf_loc (MSG_NOTE, vect_location,
6431 : "transform binary/unary operation.\n");
6432 :
6433 : /* Handle def. */
6434 8776 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6435 :
6436 8776 : unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
6437 8776 : if (scalar_shift_arg && dt[1] != vect_internal_def)
6438 : {
6439 : /* Vector shl and shr insn patterns can be defined with scalar
6440 : operand 2 (shift operand). In this case, use constant or loop
6441 : invariant op1 directly, without extending it to vector mode
6442 : first. */
6443 6606 : optab_op2_mode = insn_data[icode].operand[2].mode;
6444 6606 : if (!VECTOR_MODE_P (optab_op2_mode))
6445 : {
6446 6606 : if (dump_enabled_p ())
6447 1920 : dump_printf_loc (MSG_NOTE, vect_location,
6448 : "operand 1 using scalar mode.\n");
6449 6606 : vec_oprnd1 = op1;
6450 6606 : vec_oprnds1.create (nvectors);
6451 6606 : vec_oprnds1.quick_push (vec_oprnd1);
6452 : /* Store vec_oprnd1 for every vector stmt to be created.
6453 : We check during the analysis that all the shift arguments
6454 : are the same.
6455 : TODO: Allow different constants for different vector
6456 : stmts generated for an SLP instance. */
6457 15275 : for (k = 0; k < nvectors - 1; k++)
6458 2063 : vec_oprnds1.quick_push (vec_oprnd1);
6459 : }
6460 : }
6461 2170 : else if (!scalar_shift_arg && incompatible_op1_vectype_p)
6462 : {
6463 0 : if (was_scalar_shift_arg)
6464 : {
6465 : /* If the argument was the same in all lanes create the
6466 : correctly typed vector shift amount directly. Note
6467 : we made SLP scheduling think we use the original scalars,
6468 : so place the compensation code next to the shift which
6469 : is conservative. See PR119640 where it otherwise breaks. */
6470 0 : op1 = fold_convert (TREE_TYPE (vectype), op1);
6471 0 : op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
6472 : gsi);
6473 0 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
6474 : gsi);
6475 0 : vec_oprnds1.create (nvectors);
6476 0 : for (k = 0; k < nvectors; k++)
6477 0 : vec_oprnds1.quick_push (vec_oprnd1);
6478 : }
6479 0 : else if (dt[1] == vect_constant_def)
6480 : /* The constant shift amount has been adjusted in place. */
6481 : ;
6482 : else
6483 0 : gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
6484 : }
6485 :
6486 : /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6487 : (a special case for certain kind of vector shifts); otherwise,
6488 : operand 1 should be of a vector type (the usual case). */
6489 8776 : vect_get_vec_defs (vinfo, slp_node,
6490 : true, &vec_oprnds0, !vec_oprnd1, &vec_oprnds1);
6491 :
6492 : /* Arguments are ready. Create the new vector stmt. */
6493 31901 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6494 : {
6495 : /* For internal defs where we need to use a scalar shift arg
6496 : extract the first lane. */
6497 14349 : if (scalar_shift_arg && dt[1] == vect_internal_def)
6498 : {
6499 10 : vop1 = vec_oprnds1[0];
6500 10 : new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
6501 10 : gassign *new_stmt
6502 10 : = gimple_build_assign (new_temp,
6503 10 : build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
6504 : vop1,
6505 10 : TYPE_SIZE (TREE_TYPE (new_temp)),
6506 : bitsize_zero_node));
6507 10 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6508 10 : vop1 = new_temp;
6509 10 : }
6510 : else
6511 14339 : vop1 = vec_oprnds1[i];
6512 14349 : gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
6513 14349 : new_temp = make_ssa_name (vec_dest, new_stmt);
6514 14349 : gimple_assign_set_lhs (new_stmt, new_temp);
6515 14349 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6516 14349 : slp_node->push_vec_def (new_stmt);
6517 : }
6518 :
6519 8776 : vec_oprnds0.release ();
6520 8776 : vec_oprnds1.release ();
6521 :
6522 8776 : return true;
6523 : }
6524 :
6525 : /* Function vectorizable_operation.
6526 :
6527 : Check if STMT_INFO performs a binary, unary or ternary operation that can
6528 : be vectorized.
6529 : If COST_VEC is passed, calculate costs but don't change anything,
6530 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6531 : it, and insert it at GSI.
6532 : Return true if STMT_INFO is vectorizable in this way. */
6533 :
6534 : static bool
6535 2661172 : vectorizable_operation (vec_info *vinfo,
6536 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6537 : slp_tree slp_node,
6538 : stmt_vector_for_cost *cost_vec)
6539 : {
6540 2661172 : tree vec_dest;
6541 2661172 : tree scalar_dest;
6542 2661172 : tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
6543 2661172 : tree vectype;
6544 2661172 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6545 2661172 : enum tree_code code, orig_code;
6546 2661172 : machine_mode vec_mode;
6547 2661172 : tree new_temp;
6548 2661172 : int op_type;
6549 2661172 : optab optab;
6550 2661172 : bool target_support_p;
6551 2661172 : enum vect_def_type dt[3]
6552 : = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
6553 2661172 : poly_uint64 nunits_in;
6554 2661172 : poly_uint64 nunits_out;
6555 2661172 : tree vectype_out;
6556 2661172 : int i;
6557 2661172 : vec<tree> vec_oprnds0 = vNULL;
6558 2661172 : vec<tree> vec_oprnds1 = vNULL;
6559 2661172 : vec<tree> vec_oprnds2 = vNULL;
6560 2661172 : tree vop0, vop1, vop2;
6561 2661172 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6562 :
6563 2661172 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6564 : return false;
6565 :
6566 2661172 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6567 244447 : && cost_vec)
6568 : return false;
6569 :
6570 : /* Is STMT a vectorizable binary/unary operation? */
6571 2416725 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6572 2340388 : if (!stmt)
6573 : return false;
6574 :
6575 : /* Loads and stores are handled in vectorizable_{load,store}. */
6576 2340388 : if (STMT_VINFO_DATA_REF (stmt_info))
6577 : return false;
6578 :
6579 941175 : orig_code = code = gimple_assign_rhs_code (stmt);
6580 :
6581 : /* Shifts are handled in vectorizable_shift. */
6582 941175 : if (code == LSHIFT_EXPR
6583 : || code == RSHIFT_EXPR
6584 : || code == LROTATE_EXPR
6585 941175 : || code == RROTATE_EXPR)
6586 : return false;
6587 :
6588 : /* Comparisons are handled in vectorizable_comparison. */
6589 890317 : if (TREE_CODE_CLASS (code) == tcc_comparison)
6590 : return false;
6591 :
6592 : /* Conditions are handled in vectorizable_condition. */
6593 690280 : if (code == COND_EXPR)
6594 : return false;
6595 :
6596 : /* For pointer addition and subtraction, we should use the normal
6597 : plus and minus for the vector operation. */
6598 663416 : if (code == POINTER_PLUS_EXPR)
6599 : code = PLUS_EXPR;
6600 643463 : if (code == POINTER_DIFF_EXPR)
6601 3027 : code = MINUS_EXPR;
6602 :
6603 : /* Support only unary or binary operations. */
6604 663416 : op_type = TREE_CODE_LENGTH (code);
6605 663416 : if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
6606 : {
6607 0 : if (dump_enabled_p ())
6608 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6609 : "num. args = %d (not unary/binary/ternary op).\n",
6610 : op_type);
6611 : return false;
6612 : }
6613 :
6614 663416 : scalar_dest = gimple_assign_lhs (stmt);
6615 663416 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6616 :
6617 : /* Most operations cannot handle bit-precision types without extra
6618 : truncations. */
6619 663416 : bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
6620 651794 : if (!mask_op_p
6621 651794 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
6622 : /* Exception are bitwise binary operations. */
6623 : && code != BIT_IOR_EXPR
6624 1936 : && code != BIT_XOR_EXPR
6625 1101 : && code != BIT_AND_EXPR)
6626 : {
6627 793 : if (dump_enabled_p ())
6628 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6629 : "bit-precision arithmetic not supported.\n");
6630 : return false;
6631 : }
6632 :
6633 662623 : slp_tree slp_op0;
6634 662623 : if (!vect_is_simple_use (vinfo, slp_node,
6635 : 0, &op0, &slp_op0, &dt[0], &vectype))
6636 : {
6637 0 : if (dump_enabled_p ())
6638 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6639 : "use not simple.\n");
6640 : return false;
6641 : }
6642 662623 : bool is_invariant = (dt[0] == vect_external_def
6643 662623 : || dt[0] == vect_constant_def);
6644 : /* If op0 is an external or constant def, infer the vector type
6645 : from the scalar type. */
6646 662623 : if (!vectype)
6647 : {
6648 : /* For boolean type we cannot determine vectype by
6649 : invariant value (don't know whether it is a vector
6650 : of booleans or vector of integers). We use output
6651 : vectype because operations on boolean don't change
6652 : type. */
6653 78356 : if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
6654 : {
6655 2057 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
6656 : {
6657 317 : if (dump_enabled_p ())
6658 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6659 : "not supported operation on bool value.\n");
6660 : return false;
6661 : }
6662 1740 : vectype = vectype_out;
6663 : }
6664 : else
6665 76299 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
6666 : slp_node);
6667 : }
6668 662306 : if (!cost_vec)
6669 116938 : gcc_assert (vectype);
6670 662306 : if (!vectype)
6671 : {
6672 292 : if (dump_enabled_p ())
6673 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6674 : "no vectype for scalar type %T\n",
6675 2 : TREE_TYPE (op0));
6676 :
6677 : return false;
6678 : }
6679 :
6680 662014 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6681 662014 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6682 662014 : if (maybe_ne (nunits_out, nunits_in)
6683 662014 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
6684 : return false;
6685 :
6686 640930 : tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
6687 640930 : slp_tree slp_op1 = NULL, slp_op2 = NULL;
6688 640930 : if (op_type == binary_op || op_type == ternary_op)
6689 : {
6690 559281 : if (!vect_is_simple_use (vinfo, slp_node,
6691 : 1, &op1, &slp_op1, &dt[1], &vectype2))
6692 : {
6693 0 : if (dump_enabled_p ())
6694 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6695 : "use not simple.\n");
6696 : return false;
6697 : }
6698 559281 : is_invariant &= (dt[1] == vect_external_def
6699 559281 : || dt[1] == vect_constant_def);
6700 559281 : if (vectype2
6701 906134 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
6702 346853 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6703 346853 : TREE_TYPE (vectype2))))
6704 : return false;
6705 : }
6706 640922 : if (op_type == ternary_op)
6707 : {
6708 0 : if (!vect_is_simple_use (vinfo, slp_node,
6709 : 2, &op2, &slp_op2, &dt[2], &vectype3))
6710 : {
6711 0 : if (dump_enabled_p ())
6712 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6713 : "use not simple.\n");
6714 : return false;
6715 : }
6716 0 : is_invariant &= (dt[2] == vect_external_def
6717 0 : || dt[2] == vect_constant_def);
6718 0 : if (vectype3
6719 0 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
6720 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6721 0 : TREE_TYPE (vectype3))))
6722 : return false;
6723 : }
6724 :
6725 : /* Multiple types in SLP are handled by creating the appropriate number of
6726 : vectorized stmts for each SLP node. */
6727 640922 : auto vec_num = vect_get_num_copies (vinfo, slp_node);
6728 :
6729 : /* Reject attempts to combine mask types with nonmask types, e.g. if
6730 : we have an AND between a (nonmask) boolean loaded from memory and
6731 : a (mask) boolean result of a comparison.
6732 :
6733 : TODO: We could easily fix these cases up using pattern statements. */
6734 640922 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
6735 979543 : || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
6736 1281844 : || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
6737 : {
6738 0 : if (dump_enabled_p ())
6739 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6740 : "mixed mask and nonmask vector types\n");
6741 : return false;
6742 : }
6743 :
6744 : /* Supportable by target? */
6745 :
6746 640922 : vec_mode = TYPE_MODE (vectype);
6747 640922 : optab = optab_for_tree_code (code, vectype, optab_default);
6748 640922 : if (!optab)
6749 : {
6750 69895 : if (dump_enabled_p ())
6751 6024 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6752 : "no optab for %s and %T.\n",
6753 : get_tree_code_name (code), vectype);
6754 : return false;
6755 : }
6756 571027 : target_support_p = can_implement_p (optab, vec_mode);
6757 :
6758 571027 : bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
6759 571027 : if (!target_support_p || using_emulated_vectors_p)
6760 : {
6761 30812 : if (dump_enabled_p ())
6762 1152 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6763 : "op not supported by target.\n");
6764 : /* When vec_mode is not a vector mode and we verified ops we
6765 : do not have to lower like AND are natively supported let
6766 : those through even when the mode isn't word_mode. For
6767 : ops we have to lower the lowering code assumes we are
6768 : dealing with word_mode. */
6769 61624 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
6770 30670 : || !GET_MODE_SIZE (vec_mode).is_constant ()
6771 30670 : || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6772 25488 : || !target_support_p)
6773 65870 : && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
6774 : /* Check only during analysis. */
6775 43073 : || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
6776 : {
6777 30204 : if (dump_enabled_p ())
6778 1150 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
6779 : return false;
6780 : }
6781 608 : if (dump_enabled_p ())
6782 2 : dump_printf_loc (MSG_NOTE, vect_location,
6783 : "proceeding using word mode.\n");
6784 : using_emulated_vectors_p = true;
6785 : }
6786 :
6787 540823 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
6788 540823 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
6789 436323 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
6790 540823 : internal_fn cond_fn = get_conditional_internal_fn (code);
6791 540823 : internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
6792 :
6793 : /* If operating on inactive elements could generate spurious traps,
6794 : we need to restrict the operation to active lanes. Note that this
6795 : specifically doesn't apply to unhoisted invariants, since they
6796 : operate on the same value for every lane.
6797 :
6798 : Similarly, if this operation is part of a reduction, a fully-masked
6799 : loop should only change the active lanes of the reduction chain,
6800 : keeping the inactive lanes as-is. */
6801 512023 : bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
6802 983400 : || reduc_idx >= 0);
6803 :
6804 540823 : if (cost_vec) /* transformation not required. */
6805 : {
6806 423885 : if (loop_vinfo
6807 332796 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
6808 89970 : && mask_out_inactive)
6809 : {
6810 20426 : if (cond_len_fn != IFN_LAST
6811 20426 : && direct_internal_fn_supported_p (cond_len_fn, vectype,
6812 : OPTIMIZE_FOR_SPEED))
6813 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
6814 : 1);
6815 20426 : else if (cond_fn != IFN_LAST
6816 20426 : && direct_internal_fn_supported_p (cond_fn, vectype,
6817 : OPTIMIZE_FOR_SPEED))
6818 8532 : vect_record_loop_mask (loop_vinfo, masks, vec_num,
6819 : vectype, NULL);
6820 : else
6821 : {
6822 11894 : if (dump_enabled_p ())
6823 598 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6824 : "can't use a fully-masked loop because no"
6825 : " conditional operation is available.\n");
6826 11894 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6827 : }
6828 : }
6829 :
6830 : /* Put types on constant and invariant SLP children. */
6831 423885 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6832 423832 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
6833 847621 : || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
6834 : {
6835 149 : if (dump_enabled_p ())
6836 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6837 : "incompatible vector types for invariants\n");
6838 : return false;
6839 : }
6840 :
6841 423736 : SLP_TREE_TYPE (slp_node) = op_vec_info_type;
6842 423736 : DUMP_VECT_SCOPE ("vectorizable_operation");
6843 423736 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6844 423736 : if (using_emulated_vectors_p)
6845 : {
6846 : /* The above vect_model_simple_cost call handles constants
6847 : in the prologue and (mis-)costs one of the stmts as
6848 : vector stmt. See below for the actual lowering that will
6849 : be applied. */
6850 606 : unsigned n = vect_get_num_copies (vinfo, slp_node);
6851 606 : switch (code)
6852 : {
6853 217 : case PLUS_EXPR:
6854 217 : n *= 5;
6855 217 : break;
6856 352 : case MINUS_EXPR:
6857 352 : n *= 6;
6858 352 : break;
6859 0 : case NEGATE_EXPR:
6860 0 : n *= 4;
6861 0 : break;
6862 : default:
6863 : /* Bit operations do not have extra cost and are accounted
6864 : as vector stmt by vect_model_simple_cost. */
6865 : n = 0;
6866 : break;
6867 : }
6868 569 : if (n != 0)
6869 : {
6870 : /* We also need to materialize two large constants. */
6871 569 : record_stmt_cost (cost_vec, 2, scalar_stmt, stmt_info,
6872 : 0, vect_prologue);
6873 569 : record_stmt_cost (cost_vec, n, scalar_stmt, stmt_info,
6874 : 0, vect_body);
6875 : }
6876 : }
6877 423736 : return true;
6878 : }
6879 :
6880 : /* Transform. */
6881 :
6882 116938 : if (dump_enabled_p ())
6883 16556 : dump_printf_loc (MSG_NOTE, vect_location,
6884 : "transform binary/unary operation.\n");
6885 :
6886 116938 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
6887 103527 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
6888 :
6889 : /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6890 : vectors with unsigned elements, but the result is signed. So, we
6891 : need to compute the MINUS_EXPR into vectype temporary and
6892 : VIEW_CONVERT_EXPR it into the final vectype_out result. */
6893 116938 : tree vec_cvt_dest = NULL_TREE;
6894 116938 : if (orig_code == POINTER_DIFF_EXPR)
6895 : {
6896 123 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6897 123 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6898 : }
6899 : /* For reduction operations with undefined overflow behavior make sure to
6900 : pun them to unsigned since we change the order of evaluation.
6901 : ??? Avoid for in-order reductions? */
6902 116815 : else if (arith_code_with_undefined_signed_overflow (orig_code)
6903 99566 : && ANY_INTEGRAL_TYPE_P (vectype)
6904 48203 : && TYPE_OVERFLOW_UNDEFINED (vectype)
6905 142612 : && SLP_TREE_REDUC_IDX (slp_node) != -1)
6906 : {
6907 2480 : gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
6908 : || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
6909 2480 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6910 2480 : vectype = unsigned_type_for (vectype);
6911 2480 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6912 : }
6913 : /* Handle def. */
6914 : else
6915 114335 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
6916 :
6917 116938 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
6918 : slp_op1, &vec_oprnds1, slp_op2, &vec_oprnds2);
6919 : /* Arguments are ready. Create the new vector stmt. */
6920 375171 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6921 : {
6922 141295 : gimple *new_stmt = NULL;
6923 282590 : vop1 = ((op_type == binary_op || op_type == ternary_op)
6924 141295 : ? vec_oprnds1[i] : NULL_TREE);
6925 141295 : vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
6926 :
6927 141295 : if (vec_cvt_dest
6928 141295 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
6929 : {
6930 2933 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
6931 2933 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6932 : new_temp);
6933 2933 : new_temp = make_ssa_name (vec_dest, new_stmt);
6934 2933 : gimple_assign_set_lhs (new_stmt, new_temp);
6935 2933 : vect_finish_stmt_generation (vinfo, stmt_info,
6936 : new_stmt, gsi);
6937 2933 : vop0 = new_temp;
6938 : }
6939 141295 : if (vop1
6940 138533 : && vec_cvt_dest
6941 144366 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
6942 : {
6943 2933 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
6944 2933 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6945 : new_temp);
6946 2933 : new_temp = make_ssa_name (vec_dest, new_stmt);
6947 2933 : gimple_assign_set_lhs (new_stmt, new_temp);
6948 2933 : vect_finish_stmt_generation (vinfo, stmt_info,
6949 : new_stmt, gsi);
6950 2933 : vop1 = new_temp;
6951 : }
6952 141295 : if (vop2
6953 0 : && vec_cvt_dest
6954 141295 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
6955 : {
6956 0 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
6957 0 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6958 : new_temp);
6959 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6960 0 : gimple_assign_set_lhs (new_stmt, new_temp);
6961 0 : vect_finish_stmt_generation (vinfo, stmt_info,
6962 : new_stmt, gsi);
6963 0 : vop2 = new_temp;
6964 : }
6965 :
6966 141295 : if (using_emulated_vectors_p)
6967 : {
6968 : /* Lower the operation. This follows vector lowering. */
6969 2 : tree word_type = build_nonstandard_integer_type
6970 2 : (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
6971 2 : tree wvop0 = make_ssa_name (word_type);
6972 2 : new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
6973 : build1 (VIEW_CONVERT_EXPR,
6974 : word_type, vop0));
6975 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6976 2 : tree wvop1 = NULL_TREE;
6977 2 : if (vop1)
6978 : {
6979 2 : wvop1 = make_ssa_name (word_type);
6980 2 : new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
6981 : build1 (VIEW_CONVERT_EXPR,
6982 : word_type, vop1));
6983 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6984 : }
6985 :
6986 2 : tree result_low;
6987 2 : if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6988 : {
6989 1 : unsigned int width = vector_element_bits (vectype);
6990 1 : tree inner_type = TREE_TYPE (vectype);
6991 1 : HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
6992 1 : tree low_bits
6993 1 : = build_replicated_int_cst (word_type, width, max >> 1);
6994 1 : tree high_bits
6995 2 : = build_replicated_int_cst (word_type,
6996 1 : width, max & ~(max >> 1));
6997 1 : tree signs;
6998 1 : if (code == PLUS_EXPR || code == MINUS_EXPR)
6999 : {
7000 1 : signs = make_ssa_name (word_type);
7001 1 : new_stmt = gimple_build_assign (signs,
7002 : BIT_XOR_EXPR, wvop0, wvop1);
7003 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7004 1 : tree b_low = make_ssa_name (word_type);
7005 1 : new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
7006 : wvop1, low_bits);
7007 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7008 1 : tree a_low = make_ssa_name (word_type);
7009 1 : if (code == PLUS_EXPR)
7010 1 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7011 : wvop0, low_bits);
7012 : else
7013 0 : new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
7014 : wvop0, high_bits);
7015 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7016 1 : if (code == MINUS_EXPR)
7017 : {
7018 0 : new_stmt = gimple_build_assign (NULL_TREE,
7019 : BIT_NOT_EXPR, signs);
7020 0 : signs = make_ssa_name (word_type);
7021 0 : gimple_assign_set_lhs (new_stmt, signs);
7022 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7023 : new_stmt, gsi);
7024 : }
7025 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7026 : signs, high_bits);
7027 1 : signs = make_ssa_name (word_type);
7028 1 : gimple_assign_set_lhs (new_stmt, signs);
7029 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7030 1 : result_low = make_ssa_name (word_type);
7031 1 : new_stmt = gimple_build_assign (result_low, code,
7032 : a_low, b_low);
7033 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7034 : }
7035 : else /* if (code == NEGATE_EXPR) */
7036 : {
7037 0 : tree a_low = make_ssa_name (word_type);
7038 0 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7039 : wvop0, low_bits);
7040 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7041 0 : signs = make_ssa_name (word_type);
7042 0 : new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
7043 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7044 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7045 : signs, high_bits);
7046 0 : signs = make_ssa_name (word_type);
7047 0 : gimple_assign_set_lhs (new_stmt, signs);
7048 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7049 0 : result_low = make_ssa_name (word_type);
7050 0 : new_stmt = gimple_build_assign (result_low,
7051 : MINUS_EXPR, high_bits, a_low);
7052 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7053 : }
7054 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
7055 : result_low, signs);
7056 1 : result_low = make_ssa_name (word_type);
7057 1 : gimple_assign_set_lhs (new_stmt, result_low);
7058 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7059 : }
7060 : else
7061 : {
7062 1 : new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
7063 1 : result_low = make_ssa_name (word_type);
7064 1 : gimple_assign_set_lhs (new_stmt, result_low);
7065 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7066 :
7067 : }
7068 2 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
7069 : build1 (VIEW_CONVERT_EXPR,
7070 : vectype, result_low));
7071 2 : new_temp = make_ssa_name (vectype);
7072 2 : gimple_assign_set_lhs (new_stmt, new_temp);
7073 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7074 : }
7075 141293 : else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
7076 : {
7077 16 : tree mask;
7078 16 : if (masked_loop_p)
7079 16 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7080 : vec_num, vectype, i);
7081 : else
7082 : /* Dummy mask. */
7083 0 : mask = build_minus_one_cst (truth_type_for (vectype));
7084 16 : auto_vec<tree> vops (6);
7085 16 : vops.quick_push (mask);
7086 16 : vops.quick_push (vop0);
7087 16 : if (vop1)
7088 16 : vops.quick_push (vop1);
7089 16 : if (vop2)
7090 0 : vops.quick_push (vop2);
7091 16 : if (reduc_idx >= 0)
7092 : {
7093 : /* Perform the operation on active elements only and take
7094 : inactive elements from the reduction chain input. */
7095 8 : gcc_assert (!vop2);
7096 8 : vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
7097 : }
7098 : else
7099 : {
7100 8 : auto else_value = targetm.preferred_else_value
7101 8 : (cond_fn, vectype, vops.length () - 1, &vops[1]);
7102 8 : vops.quick_push (else_value);
7103 : }
7104 16 : if (len_loop_p)
7105 : {
7106 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
7107 0 : vec_num, vectype, i, 1, true);
7108 0 : signed char biasval
7109 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
7110 0 : tree bias = build_int_cst (intQI_type_node, biasval);
7111 0 : vops.quick_push (len);
7112 0 : vops.quick_push (bias);
7113 : }
7114 16 : gcall *call
7115 16 : = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
7116 : : cond_len_fn,
7117 : vops);
7118 16 : new_temp = make_ssa_name (vec_dest, call);
7119 16 : gimple_call_set_lhs (call, new_temp);
7120 16 : gimple_call_set_nothrow (call, true);
7121 16 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
7122 16 : new_stmt = call;
7123 16 : }
7124 : else
7125 : {
7126 141277 : tree mask = NULL_TREE;
7127 : /* When combining two masks check if either of them is elsewhere
7128 : combined with a loop mask, if that's the case we can mark that the
7129 : new combined mask doesn't need to be combined with a loop mask. */
7130 141277 : if (masked_loop_p
7131 141277 : && code == BIT_AND_EXPR
7132 141277 : && VECTOR_BOOLEAN_TYPE_P (vectype))
7133 : {
7134 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
7135 : {
7136 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7137 : vec_num, vectype, i);
7138 :
7139 0 : vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7140 : vop0, gsi);
7141 : }
7142 :
7143 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
7144 : {
7145 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7146 : vec_num, vectype, i);
7147 :
7148 0 : vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7149 : vop1, gsi);
7150 : }
7151 : }
7152 :
7153 141277 : new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
7154 141277 : new_temp = make_ssa_name (vec_dest, new_stmt);
7155 141277 : gimple_assign_set_lhs (new_stmt, new_temp);
7156 141277 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7157 141277 : if (using_emulated_vectors_p)
7158 : suppress_warning (new_stmt, OPT_Wvector_operation_performance);
7159 :
7160 : /* Enter the combined value into the vector cond hash so we don't
7161 : AND it with a loop mask again. */
7162 141277 : if (mask)
7163 0 : loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
7164 : }
7165 :
7166 141295 : if (vec_cvt_dest)
7167 : {
7168 3071 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
7169 3071 : new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
7170 : new_temp);
7171 3071 : new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
7172 3071 : gimple_assign_set_lhs (new_stmt, new_temp);
7173 3071 : vect_finish_stmt_generation (vinfo, stmt_info,
7174 : new_stmt, gsi);
7175 : }
7176 :
7177 141295 : slp_node->push_vec_def (new_stmt);
7178 : }
7179 :
7180 116938 : vec_oprnds0.release ();
7181 116938 : vec_oprnds1.release ();
7182 116938 : vec_oprnds2.release ();
7183 :
7184 116938 : return true;
7185 : }
7186 :
7187 : /* A helper function to ensure data reference DR_INFO's base alignment. */
7188 :
7189 : static void
7190 725783 : ensure_base_align (dr_vec_info *dr_info)
7191 : {
7192 : /* Alignment is only analyzed for the first element of a DR group,
7193 : use that to look at base alignment we need to enforce. */
7194 725783 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
7195 604941 : dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
7196 :
7197 725783 : gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
7198 :
7199 725783 : if (dr_info->base_misaligned)
7200 : {
7201 118160 : tree base_decl = dr_info->base_decl;
7202 :
7203 : // We should only be able to increase the alignment of a base object if
7204 : // we know what its new alignment should be at compile time.
7205 118160 : unsigned HOST_WIDE_INT align_base_to =
7206 118160 : DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
7207 :
7208 118160 : if (decl_in_symtab_p (base_decl))
7209 3259 : symtab_node::get (base_decl)->increase_alignment (align_base_to);
7210 114901 : else if (DECL_ALIGN (base_decl) < align_base_to)
7211 : {
7212 95338 : SET_DECL_ALIGN (base_decl, align_base_to);
7213 95338 : DECL_USER_ALIGN (base_decl) = 1;
7214 : }
7215 118160 : dr_info->base_misaligned = false;
7216 : }
7217 725783 : }
7218 :
7219 :
7220 : /* Function get_group_alias_ptr_type.
7221 :
7222 : Return the alias type for the group starting at FIRST_STMT_INFO. */
7223 :
7224 : static tree
7225 1672472 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
7226 : {
7227 1672472 : struct data_reference *first_dr, *next_dr;
7228 :
7229 1672472 : first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
7230 1672472 : stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
7231 3961904 : while (next_stmt_info)
7232 : {
7233 2479581 : next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
7234 4959162 : if (get_alias_set (DR_REF (first_dr))
7235 2479581 : != get_alias_set (DR_REF (next_dr)))
7236 : {
7237 190149 : if (dump_enabled_p ())
7238 30 : dump_printf_loc (MSG_NOTE, vect_location,
7239 : "conflicting alias set types.\n");
7240 190149 : return ptr_type_node;
7241 : }
7242 2289432 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7243 : }
7244 1482323 : return reference_alias_ptr_type (DR_REF (first_dr));
7245 : }
7246 :
7247 :
7248 : /* Function scan_operand_equal_p.
7249 :
7250 : Helper function for check_scan_store. Compare two references
7251 : with .GOMP_SIMD_LANE bases. */
7252 :
7253 : static bool
7254 1284 : scan_operand_equal_p (tree ref1, tree ref2)
7255 : {
7256 1284 : tree ref[2] = { ref1, ref2 };
7257 1284 : poly_int64 bitsize[2], bitpos[2];
7258 : tree offset[2], base[2];
7259 3852 : for (int i = 0; i < 2; ++i)
7260 : {
7261 2568 : machine_mode mode;
7262 2568 : int unsignedp, reversep, volatilep = 0;
7263 2568 : base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
7264 : &offset[i], &mode, &unsignedp,
7265 : &reversep, &volatilep);
7266 2568 : if (reversep || volatilep || maybe_ne (bitpos[i], 0))
7267 0 : return false;
7268 2568 : if (TREE_CODE (base[i]) == MEM_REF
7269 42 : && offset[i] == NULL_TREE
7270 2610 : && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
7271 : {
7272 42 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
7273 42 : if (is_gimple_assign (def_stmt)
7274 42 : && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
7275 42 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
7276 84 : && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
7277 : {
7278 42 : if (maybe_ne (mem_ref_offset (base[i]), 0))
7279 : return false;
7280 42 : base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
7281 42 : offset[i] = gimple_assign_rhs2 (def_stmt);
7282 : }
7283 : }
7284 : }
7285 :
7286 1284 : if (!operand_equal_p (base[0], base[1], 0))
7287 : return false;
7288 934 : if (maybe_ne (bitsize[0], bitsize[1]))
7289 : return false;
7290 934 : if (offset[0] != offset[1])
7291 : {
7292 916 : if (!offset[0] || !offset[1])
7293 : return false;
7294 916 : if (!operand_equal_p (offset[0], offset[1], 0))
7295 : {
7296 : tree step[2];
7297 0 : for (int i = 0; i < 2; ++i)
7298 : {
7299 0 : step[i] = integer_one_node;
7300 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7301 : {
7302 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7303 0 : if (is_gimple_assign (def_stmt)
7304 0 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
7305 0 : && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
7306 : == INTEGER_CST))
7307 : {
7308 0 : step[i] = gimple_assign_rhs2 (def_stmt);
7309 0 : offset[i] = gimple_assign_rhs1 (def_stmt);
7310 : }
7311 : }
7312 0 : else if (TREE_CODE (offset[i]) == MULT_EXPR)
7313 : {
7314 0 : step[i] = TREE_OPERAND (offset[i], 1);
7315 0 : offset[i] = TREE_OPERAND (offset[i], 0);
7316 : }
7317 0 : tree rhs1 = NULL_TREE;
7318 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7319 : {
7320 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7321 0 : if (gimple_assign_cast_p (def_stmt))
7322 0 : rhs1 = gimple_assign_rhs1 (def_stmt);
7323 : }
7324 0 : else if (CONVERT_EXPR_P (offset[i]))
7325 0 : rhs1 = TREE_OPERAND (offset[i], 0);
7326 0 : if (rhs1
7327 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7328 0 : && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
7329 0 : && (TYPE_PRECISION (TREE_TYPE (offset[i]))
7330 0 : >= TYPE_PRECISION (TREE_TYPE (rhs1))))
7331 0 : offset[i] = rhs1;
7332 : }
7333 0 : if (!operand_equal_p (offset[0], offset[1], 0)
7334 0 : || !operand_equal_p (step[0], step[1], 0))
7335 0 : return false;
7336 : }
7337 : }
7338 : return true;
7339 : }
7340 :
7341 :
7342 : enum scan_store_kind {
7343 : /* Normal permutation. */
7344 : scan_store_kind_perm,
7345 :
7346 : /* Whole vector left shift permutation with zero init. */
7347 : scan_store_kind_lshift_zero,
7348 :
7349 : /* Whole vector left shift permutation and VEC_COND_EXPR. */
7350 : scan_store_kind_lshift_cond
7351 : };
7352 :
7353 : /* Function check_scan_store.
7354 :
7355 : Verify if we can perform the needed permutations or whole vector shifts.
7356 : Return -1 on failure, otherwise exact log2 of vectype's nunits.
7357 : USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7358 : to do at each step. */
7359 :
7360 : static int
7361 1024 : scan_store_can_perm_p (tree vectype, tree init,
7362 : vec<enum scan_store_kind> *use_whole_vector = NULL)
7363 : {
7364 1024 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7365 1024 : unsigned HOST_WIDE_INT nunits;
7366 1024 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7367 : return -1;
7368 1024 : int units_log2 = exact_log2 (nunits);
7369 1024 : if (units_log2 <= 0)
7370 : return -1;
7371 :
7372 : int i;
7373 : enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
7374 4784 : for (i = 0; i <= units_log2; ++i)
7375 : {
7376 3760 : unsigned HOST_WIDE_INT j, k;
7377 3760 : enum scan_store_kind kind = scan_store_kind_perm;
7378 3760 : vec_perm_builder sel (nunits, nunits, 1);
7379 3760 : sel.quick_grow (nunits);
7380 3760 : if (i == units_log2)
7381 : {
7382 9728 : for (j = 0; j < nunits; ++j)
7383 8704 : sel[j] = nunits - 1;
7384 : }
7385 : else
7386 : {
7387 10416 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7388 7680 : sel[j] = j;
7389 26416 : for (k = 0; j < nunits; ++j, ++k)
7390 23680 : sel[j] = nunits + k;
7391 : }
7392 6496 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7393 3760 : if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
7394 : {
7395 0 : if (i == units_log2)
7396 : return -1;
7397 :
7398 0 : if (whole_vector_shift_kind == scan_store_kind_perm)
7399 : {
7400 0 : if (!can_implement_p (vec_shl_optab, vec_mode))
7401 : return -1;
7402 0 : whole_vector_shift_kind = scan_store_kind_lshift_zero;
7403 : /* Whole vector shifts shift in zeros, so if init is all zero
7404 : constant, there is no need to do anything further. */
7405 0 : if ((TREE_CODE (init) != INTEGER_CST
7406 0 : && TREE_CODE (init) != REAL_CST)
7407 0 : || !initializer_zerop (init))
7408 : {
7409 0 : tree masktype = truth_type_for (vectype);
7410 0 : if (!expand_vec_cond_expr_p (vectype, masktype))
7411 : return -1;
7412 : whole_vector_shift_kind = scan_store_kind_lshift_cond;
7413 : }
7414 : }
7415 0 : kind = whole_vector_shift_kind;
7416 : }
7417 3760 : if (use_whole_vector)
7418 : {
7419 1880 : if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
7420 0 : use_whole_vector->safe_grow_cleared (i, true);
7421 5640 : if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
7422 0 : use_whole_vector->safe_push (kind);
7423 : }
7424 3760 : }
7425 :
7426 : return units_log2;
7427 : }
7428 :
7429 :
7430 : /* Function check_scan_store.
7431 :
7432 : Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7433 :
7434 : static bool
7435 1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
7436 : enum vect_def_type rhs_dt, slp_tree slp_node,
7437 : slp_tree mask_node,
7438 : vect_memory_access_type memory_access_type)
7439 : {
7440 1076 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7441 1076 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7442 1076 : tree ref_type;
7443 :
7444 1076 : gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
7445 1076 : if (SLP_TREE_LANES (slp_node) > 1
7446 1076 : || mask_node
7447 1076 : || memory_access_type != VMAT_CONTIGUOUS
7448 1076 : || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
7449 1076 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
7450 1076 : || loop_vinfo == NULL
7451 1076 : || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
7452 1076 : || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
7453 1076 : || STMT_VINFO_GROUPED_ACCESS (stmt_info)
7454 1076 : || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
7455 1076 : || !integer_zerop (DR_INIT (dr_info->dr))
7456 1076 : || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
7457 2152 : || !alias_sets_conflict_p (get_alias_set (vectype),
7458 1076 : get_alias_set (TREE_TYPE (ref_type))))
7459 : {
7460 0 : if (dump_enabled_p ())
7461 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7462 : "unsupported OpenMP scan store.\n");
7463 : return false;
7464 : }
7465 :
7466 : /* We need to pattern match code built by OpenMP lowering and simplified
7467 : by following optimizations into something we can handle.
7468 : #pragma omp simd reduction(inscan,+:r)
7469 : for (...)
7470 : {
7471 : r += something ();
7472 : #pragma omp scan inclusive (r)
7473 : use (r);
7474 : }
7475 : shall have body with:
7476 : // Initialization for input phase, store the reduction initializer:
7477 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7478 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7479 : D.2042[_21] = 0;
7480 : // Actual input phase:
7481 : ...
7482 : r.0_5 = D.2042[_20];
7483 : _6 = _4 + r.0_5;
7484 : D.2042[_20] = _6;
7485 : // Initialization for scan phase:
7486 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7487 : _26 = D.2043[_25];
7488 : _27 = D.2042[_25];
7489 : _28 = _26 + _27;
7490 : D.2043[_25] = _28;
7491 : D.2042[_25] = _28;
7492 : // Actual scan phase:
7493 : ...
7494 : r.1_8 = D.2042[_20];
7495 : ...
7496 : The "omp simd array" variable D.2042 holds the privatized copy used
7497 : inside of the loop and D.2043 is another one that holds copies of
7498 : the current original list item. The separate GOMP_SIMD_LANE ifn
7499 : kinds are there in order to allow optimizing the initializer store
7500 : and combiner sequence, e.g. if it is originally some C++ish user
7501 : defined reduction, but allow the vectorizer to pattern recognize it
7502 : and turn into the appropriate vectorized scan.
7503 :
7504 : For exclusive scan, this is slightly different:
7505 : #pragma omp simd reduction(inscan,+:r)
7506 : for (...)
7507 : {
7508 : use (r);
7509 : #pragma omp scan exclusive (r)
7510 : r += something ();
7511 : }
7512 : shall have body with:
7513 : // Initialization for input phase, store the reduction initializer:
7514 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7515 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7516 : D.2042[_21] = 0;
7517 : // Actual input phase:
7518 : ...
7519 : r.0_5 = D.2042[_20];
7520 : _6 = _4 + r.0_5;
7521 : D.2042[_20] = _6;
7522 : // Initialization for scan phase:
7523 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7524 : _26 = D.2043[_25];
7525 : D.2044[_25] = _26;
7526 : _27 = D.2042[_25];
7527 : _28 = _26 + _27;
7528 : D.2043[_25] = _28;
7529 : // Actual scan phase:
7530 : ...
7531 : r.1_8 = D.2044[_20];
7532 : ... */
7533 :
7534 1076 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
7535 : {
7536 : /* Match the D.2042[_21] = 0; store above. Just require that
7537 : it is a constant or external definition store. */
7538 564 : if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
7539 : {
7540 0 : fail_init:
7541 0 : if (dump_enabled_p ())
7542 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7543 : "unsupported OpenMP scan initializer store.\n");
7544 : return false;
7545 : }
7546 :
7547 564 : if (! loop_vinfo->scan_map)
7548 322 : loop_vinfo->scan_map = new hash_map<tree, tree>;
7549 564 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7550 564 : tree &cached = loop_vinfo->scan_map->get_or_insert (var);
7551 564 : if (cached)
7552 0 : goto fail_init;
7553 564 : cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
7554 :
7555 : /* These stores can be vectorized normally. */
7556 564 : return true;
7557 : }
7558 :
7559 512 : if (rhs_dt != vect_internal_def)
7560 : {
7561 0 : fail:
7562 0 : if (dump_enabled_p ())
7563 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7564 : "unsupported OpenMP scan combiner pattern.\n");
7565 : return false;
7566 : }
7567 :
7568 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7569 512 : tree rhs = gimple_assign_rhs1 (stmt);
7570 512 : if (TREE_CODE (rhs) != SSA_NAME)
7571 0 : goto fail;
7572 :
7573 512 : gimple *other_store_stmt = NULL;
7574 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7575 512 : bool inscan_var_store
7576 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7577 :
7578 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7579 : {
7580 252 : if (!inscan_var_store)
7581 : {
7582 126 : use_operand_p use_p;
7583 126 : imm_use_iterator iter;
7584 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7585 : {
7586 252 : gimple *use_stmt = USE_STMT (use_p);
7587 252 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7588 126 : continue;
7589 126 : if (gimple_bb (use_stmt) != gimple_bb (stmt)
7590 126 : || !is_gimple_assign (use_stmt)
7591 126 : || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
7592 126 : || other_store_stmt
7593 252 : || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
7594 0 : goto fail;
7595 126 : other_store_stmt = use_stmt;
7596 0 : }
7597 126 : if (other_store_stmt == NULL)
7598 0 : goto fail;
7599 126 : rhs = gimple_assign_lhs (other_store_stmt);
7600 126 : if (!single_imm_use (rhs, &use_p, &other_store_stmt))
7601 0 : goto fail;
7602 : }
7603 : }
7604 260 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
7605 : {
7606 260 : use_operand_p use_p;
7607 260 : imm_use_iterator iter;
7608 780 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7609 : {
7610 520 : gimple *use_stmt = USE_STMT (use_p);
7611 520 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7612 260 : continue;
7613 260 : if (other_store_stmt)
7614 0 : goto fail;
7615 260 : other_store_stmt = use_stmt;
7616 260 : }
7617 : }
7618 : else
7619 0 : goto fail;
7620 :
7621 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7622 512 : if (gimple_bb (def_stmt) != gimple_bb (stmt)
7623 512 : || !is_gimple_assign (def_stmt)
7624 1024 : || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
7625 0 : goto fail;
7626 :
7627 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7628 : /* For pointer addition, we should use the normal plus for the vector
7629 : operation. */
7630 512 : switch (code)
7631 : {
7632 0 : case POINTER_PLUS_EXPR:
7633 0 : code = PLUS_EXPR;
7634 0 : break;
7635 0 : case MULT_HIGHPART_EXPR:
7636 0 : goto fail;
7637 : default:
7638 : break;
7639 : }
7640 512 : if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
7641 0 : goto fail;
7642 :
7643 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7644 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7645 512 : if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
7646 0 : goto fail;
7647 :
7648 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7649 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7650 512 : if (gimple_bb (load1_stmt) != gimple_bb (stmt)
7651 512 : || !gimple_assign_load_p (load1_stmt)
7652 512 : || gimple_bb (load2_stmt) != gimple_bb (stmt)
7653 1024 : || !gimple_assign_load_p (load2_stmt))
7654 0 : goto fail;
7655 :
7656 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7657 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7658 512 : if (load1_stmt_info == NULL
7659 512 : || load2_stmt_info == NULL
7660 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
7661 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
7662 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
7663 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7664 0 : goto fail;
7665 :
7666 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
7667 : {
7668 126 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7669 126 : if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
7670 126 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
7671 0 : goto fail;
7672 126 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7673 126 : tree lrhs;
7674 126 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7675 : lrhs = rhs1;
7676 : else
7677 16 : lrhs = rhs2;
7678 126 : use_operand_p use_p;
7679 126 : imm_use_iterator iter;
7680 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
7681 : {
7682 252 : gimple *use_stmt = USE_STMT (use_p);
7683 252 : if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
7684 126 : continue;
7685 126 : if (other_store_stmt)
7686 0 : goto fail;
7687 126 : other_store_stmt = use_stmt;
7688 126 : }
7689 : }
7690 :
7691 512 : if (other_store_stmt == NULL)
7692 0 : goto fail;
7693 512 : if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
7694 512 : || !gimple_store_p (other_store_stmt))
7695 0 : goto fail;
7696 :
7697 512 : stmt_vec_info other_store_stmt_info
7698 512 : = loop_vinfo->lookup_stmt (other_store_stmt);
7699 512 : if (other_store_stmt_info == NULL
7700 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
7701 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7702 0 : goto fail;
7703 :
7704 512 : gimple *stmt1 = stmt;
7705 512 : gimple *stmt2 = other_store_stmt;
7706 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7707 : std::swap (stmt1, stmt2);
7708 512 : if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
7709 : gimple_assign_rhs1 (load2_stmt)))
7710 : {
7711 162 : std::swap (rhs1, rhs2);
7712 162 : std::swap (load1_stmt, load2_stmt);
7713 162 : std::swap (load1_stmt_info, load2_stmt_info);
7714 : }
7715 512 : if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
7716 : gimple_assign_rhs1 (load1_stmt)))
7717 0 : goto fail;
7718 :
7719 512 : tree var3 = NULL_TREE;
7720 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
7721 512 : && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
7722 : gimple_assign_rhs1 (load2_stmt)))
7723 0 : goto fail;
7724 512 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7725 : {
7726 252 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7727 252 : if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
7728 252 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
7729 0 : goto fail;
7730 252 : var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7731 252 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
7732 252 : || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
7733 504 : || lookup_attribute ("omp simd inscan exclusive",
7734 252 : DECL_ATTRIBUTES (var3)))
7735 0 : goto fail;
7736 : }
7737 :
7738 512 : dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
7739 512 : if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
7740 512 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
7741 0 : goto fail;
7742 :
7743 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7744 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
7745 512 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
7746 512 : || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
7747 1024 : || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7748 512 : == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
7749 0 : goto fail;
7750 :
7751 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7752 256 : std::swap (var1, var2);
7753 :
7754 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7755 : {
7756 252 : if (!lookup_attribute ("omp simd inscan exclusive",
7757 252 : DECL_ATTRIBUTES (var1)))
7758 0 : goto fail;
7759 252 : var1 = var3;
7760 : }
7761 :
7762 512 : if (loop_vinfo->scan_map == NULL)
7763 0 : goto fail;
7764 512 : tree *init = loop_vinfo->scan_map->get (var1);
7765 512 : if (init == NULL)
7766 0 : goto fail;
7767 :
7768 : /* The IL is as expected, now check if we can actually vectorize it.
7769 : Inclusive scan:
7770 : _26 = D.2043[_25];
7771 : _27 = D.2042[_25];
7772 : _28 = _26 + _27;
7773 : D.2043[_25] = _28;
7774 : D.2042[_25] = _28;
7775 : should be vectorized as (where _40 is the vectorized rhs
7776 : from the D.2042[_21] = 0; store):
7777 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7778 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7779 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7780 : _33 = _31 + _32;
7781 : // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7782 : _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7783 : _35 = _33 + _34;
7784 : // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7785 : // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7786 : _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7787 : _37 = _35 + _36;
7788 : // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7789 : // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7790 : _38 = _30 + _37;
7791 : _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7792 : MEM <vector(8) int> [(int *)&D.2043] = _39;
7793 : MEM <vector(8) int> [(int *)&D.2042] = _38;
7794 : Exclusive scan:
7795 : _26 = D.2043[_25];
7796 : D.2044[_25] = _26;
7797 : _27 = D.2042[_25];
7798 : _28 = _26 + _27;
7799 : D.2043[_25] = _28;
7800 : should be vectorized as (where _40 is the vectorized rhs
7801 : from the D.2042[_21] = 0; store):
7802 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7803 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7804 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7805 : _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7806 : _34 = _32 + _33;
7807 : // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7808 : // _31[3]+_31[4], ... _31[5]+.._31[6] };
7809 : _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7810 : _36 = _34 + _35;
7811 : // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7812 : // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7813 : _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7814 : _38 = _36 + _37;
7815 : // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7816 : // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7817 : _39 = _30 + _38;
7818 : _50 = _31 + _39;
7819 : _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7820 : MEM <vector(8) int> [(int *)&D.2044] = _39;
7821 : MEM <vector(8) int> [(int *)&D.2042] = _51; */
7822 512 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7823 512 : optab optab = optab_for_tree_code (code, vectype, optab_default);
7824 512 : if (!optab || !can_implement_p (optab, vec_mode))
7825 0 : goto fail;
7826 :
7827 512 : int units_log2 = scan_store_can_perm_p (vectype, *init);
7828 512 : if (units_log2 == -1)
7829 0 : goto fail;
7830 :
7831 : return true;
7832 : }
7833 :
7834 :
7835 : /* Function vectorizable_scan_store.
7836 :
7837 : Helper of vectorizable_score, arguments like on vectorizable_store.
7838 : Handle only the transformation, checking is done in check_scan_store. */
7839 :
7840 : static bool
7841 512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
7842 : slp_tree slp_node, gimple_stmt_iterator *gsi)
7843 : {
7844 512 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7845 512 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7846 512 : tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
7847 512 : tree vectype = SLP_TREE_VECTYPE (slp_node);
7848 :
7849 512 : if (dump_enabled_p ())
7850 492 : dump_printf_loc (MSG_NOTE, vect_location,
7851 : "transform scan store.\n");
7852 :
7853 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7854 512 : tree rhs = gimple_assign_rhs1 (stmt);
7855 512 : gcc_assert (TREE_CODE (rhs) == SSA_NAME);
7856 :
7857 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7858 512 : bool inscan_var_store
7859 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7860 :
7861 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7862 : {
7863 126 : use_operand_p use_p;
7864 126 : imm_use_iterator iter;
7865 126 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7866 : {
7867 126 : gimple *use_stmt = USE_STMT (use_p);
7868 126 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7869 0 : continue;
7870 126 : rhs = gimple_assign_lhs (use_stmt);
7871 126 : break;
7872 126 : }
7873 : }
7874 :
7875 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7876 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7877 512 : if (code == POINTER_PLUS_EXPR)
7878 0 : code = PLUS_EXPR;
7879 512 : gcc_assert (TREE_CODE_LENGTH (code) == binary_op
7880 : && commutative_tree_code (code));
7881 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7882 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7883 512 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
7884 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7885 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7886 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7887 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7888 512 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7889 512 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7890 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7891 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7892 :
7893 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7894 : {
7895 436 : std::swap (rhs1, rhs2);
7896 436 : std::swap (var1, var2);
7897 436 : std::swap (load1_dr_info, load2_dr_info);
7898 : }
7899 :
7900 512 : tree *init = loop_vinfo->scan_map->get (var1);
7901 512 : gcc_assert (init);
7902 :
7903 512 : unsigned HOST_WIDE_INT nunits;
7904 512 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7905 : gcc_unreachable ();
7906 512 : auto_vec<enum scan_store_kind, 16> use_whole_vector;
7907 512 : int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
7908 512 : gcc_assert (units_log2 > 0);
7909 512 : auto_vec<tree, 16> perms;
7910 512 : perms.quick_grow (units_log2 + 1);
7911 512 : tree zero_vec = NULL_TREE, masktype = NULL_TREE;
7912 2904 : for (int i = 0; i <= units_log2; ++i)
7913 : {
7914 1880 : unsigned HOST_WIDE_INT j, k;
7915 1880 : vec_perm_builder sel (nunits, nunits, 1);
7916 1880 : sel.quick_grow (nunits);
7917 1880 : if (i == units_log2)
7918 4864 : for (j = 0; j < nunits; ++j)
7919 4352 : sel[j] = nunits - 1;
7920 : else
7921 : {
7922 5208 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7923 3840 : sel[j] = j;
7924 13208 : for (k = 0; j < nunits; ++j, ++k)
7925 11840 : sel[j] = nunits + k;
7926 : }
7927 3248 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7928 1880 : if (!use_whole_vector.is_empty ()
7929 0 : && use_whole_vector[i] != scan_store_kind_perm)
7930 : {
7931 0 : if (zero_vec == NULL_TREE)
7932 0 : zero_vec = build_zero_cst (vectype);
7933 0 : if (masktype == NULL_TREE
7934 0 : && use_whole_vector[i] == scan_store_kind_lshift_cond)
7935 0 : masktype = truth_type_for (vectype);
7936 0 : perms[i] = vect_gen_perm_mask_any (vectype, indices);
7937 : }
7938 : else
7939 1880 : perms[i] = vect_gen_perm_mask_checked (vectype, indices);
7940 1880 : }
7941 :
7942 512 : tree vec_oprnd1 = NULL_TREE;
7943 512 : tree vec_oprnd2 = NULL_TREE;
7944 512 : tree vec_oprnd3 = NULL_TREE;
7945 512 : tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
7946 512 : tree dataref_offset = build_int_cst (ref_type, 0);
7947 512 : tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
7948 512 : tree ldataref_ptr = NULL_TREE;
7949 512 : tree orig = NULL_TREE;
7950 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7951 126 : ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
7952 : /* The initialization is invariant. */
7953 512 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
7954 512 : auto_vec<tree> vec_oprnds2;
7955 512 : auto_vec<tree> vec_oprnds3;
7956 512 : if (ldataref_ptr == NULL)
7957 : {
7958 : /* We want to lookup the vector operands of the reduction, not those
7959 : of the store - for SLP we have to use the proper SLP node for the
7960 : lookup, which should be the single child of the scan store. */
7961 386 : vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
7962 : rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
7963 : /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
7964 : them in SLP child order. So we have to swap here with logic
7965 : similar to above. */
7966 386 : stmt_vec_info load
7967 386 : = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
7968 386 : (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
7969 386 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
7970 386 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7971 386 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
7972 820 : for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
7973 494 : std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
7974 : }
7975 : else
7976 126 : vect_get_vec_defs (vinfo, slp_node,
7977 : rhs2, &vec_oprnds3);
7978 1248 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
7979 : {
7980 736 : if (ldataref_ptr == NULL)
7981 554 : vec_oprnd2 = vec_oprnds2[j];
7982 736 : vec_oprnd3 = vec_oprnds3[j];
7983 736 : if (j == 0)
7984 : orig = vec_oprnd3;
7985 224 : else if (!inscan_var_store)
7986 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
7987 :
7988 736 : if (ldataref_ptr)
7989 : {
7990 182 : vec_oprnd2 = make_ssa_name (vectype);
7991 182 : tree data_ref = fold_build2 (MEM_REF, vectype,
7992 : unshare_expr (ldataref_ptr),
7993 : dataref_offset);
7994 182 : vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
7995 182 : gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
7996 182 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
7997 : }
7998 :
7999 736 : tree v = vec_oprnd2;
8000 3068 : for (int i = 0; i < units_log2; ++i)
8001 : {
8002 2332 : tree new_temp = make_ssa_name (vectype);
8003 2332 : gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
8004 : (zero_vec
8005 0 : && (use_whole_vector[i]
8006 0 : != scan_store_kind_perm))
8007 : ? zero_vec : vec_oprnd1, v,
8008 2332 : perms[i]);
8009 2332 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8010 :
8011 2332 : if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
8012 : {
8013 : /* Whole vector shift shifted in zero bits, but if *init
8014 : is not initializer_zerop, we need to replace those elements
8015 : with elements from vec_oprnd1. */
8016 0 : tree_vector_builder vb (masktype, nunits, 1);
8017 0 : for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
8018 0 : vb.quick_push (k < (HOST_WIDE_INT_1U << i)
8019 : ? boolean_false_node : boolean_true_node);
8020 :
8021 0 : tree new_temp2 = make_ssa_name (vectype);
8022 0 : g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
8023 : new_temp, vec_oprnd1);
8024 0 : vect_finish_stmt_generation (vinfo, stmt_info,
8025 : g, gsi);
8026 0 : new_temp = new_temp2;
8027 0 : }
8028 :
8029 : /* For exclusive scan, perform the perms[i] permutation once
8030 : more. */
8031 2332 : if (i == 0
8032 1100 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
8033 728 : && v == vec_oprnd2)
8034 : {
8035 364 : v = new_temp;
8036 364 : --i;
8037 364 : continue;
8038 : }
8039 :
8040 1968 : tree new_temp2 = make_ssa_name (vectype);
8041 1968 : g = gimple_build_assign (new_temp2, code, v, new_temp);
8042 1968 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8043 :
8044 1968 : v = new_temp2;
8045 : }
8046 :
8047 736 : tree new_temp = make_ssa_name (vectype);
8048 736 : gimple *g = gimple_build_assign (new_temp, code, orig, v);
8049 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8050 :
8051 736 : tree last_perm_arg = new_temp;
8052 : /* For exclusive scan, new_temp computed above is the exclusive scan
8053 : prefix sum. Turn it into inclusive prefix sum for the broadcast
8054 : of the last element into orig. */
8055 736 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
8056 : {
8057 364 : last_perm_arg = make_ssa_name (vectype);
8058 364 : g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
8059 364 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8060 : }
8061 :
8062 736 : orig = make_ssa_name (vectype);
8063 2208 : g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
8064 736 : last_perm_arg, perms[units_log2]);
8065 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8066 :
8067 736 : if (!inscan_var_store)
8068 : {
8069 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8070 : unshare_expr (dataref_ptr),
8071 : dataref_offset);
8072 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8073 368 : g = gimple_build_assign (data_ref, new_temp);
8074 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8075 : }
8076 : }
8077 :
8078 512 : if (inscan_var_store)
8079 624 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8080 : {
8081 368 : if (j != 0)
8082 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8083 :
8084 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8085 : unshare_expr (dataref_ptr),
8086 : dataref_offset);
8087 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8088 368 : gimple *g = gimple_build_assign (data_ref, orig);
8089 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8090 : }
8091 512 : return true;
8092 512 : }
8093 :
8094 :
8095 : /* Function vectorizable_store.
8096 :
8097 : Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8098 : that can be vectorized.
8099 : If COST_VEC is passed, calculate costs but don't change anything,
8100 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
8101 : it, and insert it at GSI.
8102 : Return true if STMT_INFO is vectorizable in this way. */
8103 :
8104 : static bool
8105 2149063 : vectorizable_store (vec_info *vinfo,
8106 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8107 : slp_tree slp_node,
8108 : stmt_vector_for_cost *cost_vec)
8109 : {
8110 2149063 : tree data_ref;
8111 2149063 : tree vec_oprnd = NULL_TREE;
8112 2149063 : tree elem_type;
8113 2149063 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
8114 2149063 : class loop *loop = NULL;
8115 2149063 : machine_mode vec_mode;
8116 2149063 : tree dummy;
8117 2149063 : enum vect_def_type rhs_dt = vect_unknown_def_type;
8118 2149063 : enum vect_def_type mask_dt = vect_unknown_def_type;
8119 2149063 : tree dataref_ptr = NULL_TREE;
8120 2149063 : tree dataref_offset = NULL_TREE;
8121 2149063 : int j;
8122 2149063 : stmt_vec_info first_stmt_info;
8123 2149063 : bool grouped_store;
8124 2149063 : unsigned int group_size, i;
8125 2149063 : unsigned int vec_num;
8126 2149063 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
8127 2149063 : tree aggr_type;
8128 2149063 : poly_uint64 vf;
8129 2149063 : vec_load_store_type vls_type;
8130 2149063 : tree ref_type;
8131 :
8132 2149063 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8133 : return false;
8134 :
8135 2149063 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8136 244447 : && cost_vec)
8137 : return false;
8138 :
8139 : /* Is vectorizable store? */
8140 :
8141 1904616 : tree mask_vectype = NULL_TREE;
8142 1904616 : slp_tree mask_node = NULL;
8143 1904616 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
8144 : {
8145 1829183 : tree scalar_dest = gimple_assign_lhs (assign);
8146 1829183 : if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
8147 1829183 : && is_pattern_stmt_p (stmt_info))
8148 1679 : scalar_dest = TREE_OPERAND (scalar_dest, 0);
8149 1829183 : if (TREE_CODE (scalar_dest) != ARRAY_REF
8150 1829183 : && TREE_CODE (scalar_dest) != BIT_FIELD_REF
8151 : && TREE_CODE (scalar_dest) != INDIRECT_REF
8152 : && TREE_CODE (scalar_dest) != COMPONENT_REF
8153 : && TREE_CODE (scalar_dest) != IMAGPART_EXPR
8154 : && TREE_CODE (scalar_dest) != REALPART_EXPR
8155 : && TREE_CODE (scalar_dest) != MEM_REF)
8156 : return false;
8157 : }
8158 : else
8159 : {
8160 75433 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
8161 13867 : if (!call || !gimple_call_internal_p (call))
8162 : return false;
8163 :
8164 8376 : internal_fn ifn = gimple_call_internal_fn (call);
8165 8376 : if (!internal_store_fn_p (ifn))
8166 : return false;
8167 :
8168 1857 : int mask_index = internal_fn_mask_index (ifn);
8169 1857 : if (mask_index >= 0)
8170 1857 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
8171 1857 : if (mask_index >= 0
8172 1857 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
8173 : &mask_node, &mask_dt,
8174 : &mask_vectype))
8175 : return false;
8176 : }
8177 :
8178 1387438 : tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
8179 1387438 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8180 :
8181 1387438 : if (loop_vinfo)
8182 : {
8183 228841 : loop = LOOP_VINFO_LOOP (loop_vinfo);
8184 228841 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8185 : }
8186 : else
8187 : vf = 1;
8188 1387438 : vec_num = vect_get_num_copies (vinfo, slp_node);
8189 :
8190 : /* FORNOW. This restriction should be relaxed. */
8191 1387438 : if (loop
8192 1387713 : && nested_in_vect_loop_p (loop, stmt_info)
8193 1387721 : && vec_num > 1)
8194 : {
8195 8 : if (dump_enabled_p ())
8196 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8197 : "multiple types in nested loop.\n");
8198 : return false;
8199 : }
8200 :
8201 1387430 : slp_tree op_node;
8202 1387430 : if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
8203 : &op_node, &rhs_dt, &rhs_vectype, &vls_type))
8204 : return false;
8205 :
8206 1387406 : elem_type = TREE_TYPE (vectype);
8207 1387406 : vec_mode = TYPE_MODE (vectype);
8208 :
8209 1387406 : if (!STMT_VINFO_DATA_REF (stmt_info))
8210 : return false;
8211 :
8212 1387406 : vect_load_store_data _ls_data{};
8213 1387406 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
8214 1387406 : if (cost_vec
8215 1387406 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
8216 : vls_type, &_ls_data))
8217 : return false;
8218 : /* Temporary aliases to analysis data, should not be modified through
8219 : these. */
8220 1386790 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
8221 1386790 : const dr_alignment_support alignment_support_scheme
8222 : = ls.alignment_support_scheme;
8223 1386790 : const int misalignment = ls.misalignment;
8224 1386790 : const poly_int64 poffset = ls.poffset;
8225 :
8226 1386790 : if (slp_node->ldst_lanes
8227 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
8228 : {
8229 0 : if (dump_enabled_p ())
8230 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8231 : "discovered store-lane but cannot use it.\n");
8232 : return false;
8233 : }
8234 :
8235 1386790 : if (mask_node)
8236 : {
8237 1767 : if (memory_access_type == VMAT_CONTIGUOUS)
8238 : {
8239 616 : if (!VECTOR_MODE_P (vec_mode)
8240 3018 : || !can_vec_mask_load_store_p (vec_mode,
8241 1509 : TYPE_MODE (mask_vectype), false))
8242 : return false;
8243 : }
8244 258 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
8245 258 : && (!mat_gather_scatter_p (memory_access_type)
8246 242 : || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
8247 170 : && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
8248 : {
8249 16 : if (dump_enabled_p ())
8250 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8251 : "unsupported access type for masked store.\n");
8252 : return false;
8253 : }
8254 242 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
8255 : {
8256 72 : if (dump_enabled_p ())
8257 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8258 : "unsupported masked emulated scatter.\n");
8259 : return false;
8260 : }
8261 : }
8262 : else
8263 : {
8264 : /* FORNOW. In some cases can vectorize even if data-type not supported
8265 : (e.g. - array initialization with 0). */
8266 1385023 : if (!can_implement_p (mov_optab, vec_mode))
8267 : return false;
8268 : }
8269 :
8270 1386588 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
8271 1386588 : grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
8272 2565186 : && !mat_gather_scatter_p (memory_access_type));
8273 1178598 : if (grouped_store)
8274 : {
8275 1178598 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8276 1178598 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8277 1178598 : group_size = DR_GROUP_SIZE (first_stmt_info);
8278 : }
8279 : else
8280 : {
8281 1386588 : first_stmt_info = stmt_info;
8282 1386588 : first_dr_info = dr_info;
8283 : group_size = 1;
8284 : }
8285 :
8286 1386588 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
8287 : {
8288 1076 : if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
8289 : mask_node, memory_access_type))
8290 : return false;
8291 : }
8292 :
8293 1941816 : bool costing_p = cost_vec;
8294 1385820 : if (costing_p) /* transformation not required. */
8295 : {
8296 830592 : if (loop_vinfo
8297 164603 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
8298 77352 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
8299 : vls_type, group_size, &ls,
8300 : mask_node);
8301 :
8302 830592 : if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
8303 830592 : || (mask_node
8304 1038 : && !vect_maybe_update_slp_op_vectype (mask_node,
8305 : mask_vectype)))
8306 : {
8307 0 : if (dump_enabled_p ())
8308 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8309 : "incompatible vector types for invariants\n");
8310 : return false;
8311 : }
8312 :
8313 830592 : if (dump_enabled_p ()
8314 : && memory_access_type != VMAT_ELEMENTWISE
8315 15138 : && memory_access_type != VMAT_STRIDED_SLP
8316 14480 : && memory_access_type != VMAT_INVARIANT
8317 845072 : && alignment_support_scheme != dr_aligned)
8318 5033 : dump_printf_loc (MSG_NOTE, vect_location,
8319 : "Vectorizing an unaligned access.\n");
8320 : }
8321 :
8322 : /* Transform. */
8323 :
8324 : if (!costing_p)
8325 555996 : ensure_base_align (dr_info);
8326 :
8327 1386588 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
8328 : {
8329 1024 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
8330 1024 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8331 1024 : if (costing_p)
8332 : {
8333 512 : unsigned int inside_cost = 0, prologue_cost = 0;
8334 512 : if (vls_type == VLS_STORE_INVARIANT)
8335 0 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8336 : slp_node, 0, vect_prologue);
8337 512 : vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
8338 : alignment_support_scheme, misalignment,
8339 : &inside_cost, cost_vec);
8340 :
8341 512 : if (dump_enabled_p ())
8342 492 : dump_printf_loc (MSG_NOTE, vect_location,
8343 : "vect_model_store_cost: inside_cost = %d, "
8344 : "prologue_cost = %d .\n",
8345 : inside_cost, prologue_cost);
8346 :
8347 512 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8348 512 : slp_node->data = new vect_load_store_data (std::move (ls));
8349 :
8350 512 : return true;
8351 : }
8352 512 : return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
8353 : }
8354 :
8355 : /* FORNOW */
8356 1385564 : gcc_assert (!grouped_store
8357 : || !loop
8358 : || !nested_in_vect_loop_p (loop, stmt_info));
8359 :
8360 1385564 : grouped_store = false;
8361 1385564 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
8362 1385564 : gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
8363 : || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
8364 1385564 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8365 :
8366 1385564 : ref_type = get_group_alias_ptr_type (first_stmt_info);
8367 :
8368 1385564 : if (!costing_p && dump_enabled_p ())
8369 12299 : dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
8370 :
8371 1385564 : if (memory_access_type == VMAT_ELEMENTWISE
8372 1385564 : || memory_access_type == VMAT_STRIDED_SLP)
8373 : {
8374 29452 : unsigned inside_cost = 0, prologue_cost = 0;
8375 29452 : gimple_stmt_iterator incr_gsi;
8376 29452 : bool insert_after;
8377 29452 : tree offvar = NULL_TREE;
8378 29452 : tree ivstep;
8379 29452 : tree running_off;
8380 29452 : tree stride_base, stride_step, alias_off;
8381 29452 : tree vec_oprnd = NULL_TREE;
8382 29452 : tree dr_offset;
8383 : /* Checked by get_load_store_type. */
8384 29452 : unsigned int const_nunits = nunits.to_constant ();
8385 :
8386 29452 : gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
8387 29452 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
8388 :
8389 29452 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
8390 29452 : stride_base
8391 29452 : = fold_build_pointer_plus
8392 : (DR_BASE_ADDRESS (first_dr_info->dr),
8393 : size_binop (PLUS_EXPR,
8394 : convert_to_ptrofftype (dr_offset),
8395 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
8396 29452 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
8397 :
8398 : /* For a store with loop-invariant (but other than power-of-2)
8399 : stride (i.e. not a grouped access) like so:
8400 :
8401 : for (i = 0; i < n; i += stride)
8402 : array[i] = ...;
8403 :
8404 : we generate a new induction variable and new stores from
8405 : the components of the (vectorized) rhs:
8406 :
8407 : for (j = 0; ; j += VF*stride)
8408 : vectemp = ...;
8409 : tmp1 = vectemp[0];
8410 : array[j] = tmp1;
8411 : tmp2 = vectemp[1];
8412 : array[j + stride] = tmp2;
8413 : ...
8414 : */
8415 :
8416 : /* ??? Modify local copies of alignment_support_scheme and
8417 : misalignment, but this part of analysis should be done
8418 : earlier and remembered, likewise the chosen load mode. */
8419 29452 : const dr_alignment_support tem = alignment_support_scheme;
8420 29452 : dr_alignment_support alignment_support_scheme = tem;
8421 29452 : const int tem2 = misalignment;
8422 29452 : int misalignment = tem2;
8423 :
8424 29452 : unsigned nstores = const_nunits;
8425 29452 : unsigned lnel = 1;
8426 29452 : tree ltype = elem_type;
8427 29452 : tree lvectype = vectype;
8428 29452 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
8429 29452 : if (n == const_nunits)
8430 : {
8431 2945 : int mis_align = dr_misalignment (first_dr_info, vectype);
8432 : /* With VF > 1 we advance the DR by step, if that is constant
8433 : and only aligned when performed VF times, DR alignment
8434 : analysis can analyze this as aligned since it assumes
8435 : contiguous accesses. But that is not how we code generate
8436 : here, so adjust for this. */
8437 2945 : if (maybe_gt (vf, 1u)
8438 4476 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8439 4241 : DR_TARGET_ALIGNMENT (first_dr_info)))
8440 235 : mis_align = -1;
8441 2945 : dr_alignment_support dr_align
8442 2945 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
8443 : mis_align);
8444 2945 : if (dr_align == dr_aligned
8445 2945 : || dr_align == dr_unaligned_supported)
8446 : {
8447 29452 : nstores = 1;
8448 29452 : lnel = const_nunits;
8449 29452 : ltype = vectype;
8450 29452 : lvectype = vectype;
8451 29452 : alignment_support_scheme = dr_align;
8452 29452 : misalignment = mis_align;
8453 : }
8454 : }
8455 26507 : else if (n > 1)
8456 : {
8457 1967 : nstores = const_nunits / n;
8458 1967 : lnel = n;
8459 1967 : ltype = build_vector_type (elem_type, n);
8460 1967 : lvectype = vectype;
8461 1967 : int mis_align = dr_misalignment (first_dr_info, ltype);
8462 1967 : if (maybe_gt (vf, 1u)
8463 3934 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8464 3292 : DR_TARGET_ALIGNMENT (first_dr_info)))
8465 642 : mis_align = -1;
8466 1967 : dr_alignment_support dr_align
8467 1967 : = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
8468 : mis_align);
8469 1967 : alignment_support_scheme = dr_align;
8470 1967 : misalignment = mis_align;
8471 :
8472 : /* First check if vec_extract optab doesn't support extraction
8473 : of vector elts directly. */
8474 1967 : scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
8475 1967 : machine_mode vmode;
8476 3934 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8477 3740 : || !related_vector_mode (TYPE_MODE (vectype), elmode,
8478 2161 : n).exists (&vmode)
8479 1773 : || (convert_optab_handler (vec_extract_optab,
8480 1773 : TYPE_MODE (vectype), vmode)
8481 : == CODE_FOR_nothing)
8482 1967 : || !(dr_align == dr_aligned
8483 172 : || dr_align == dr_unaligned_supported))
8484 : {
8485 : /* Try to avoid emitting an extract of vector elements
8486 : by performing the extracts using an integer type of the
8487 : same size, extracting from a vector of those and then
8488 : re-interpreting it as the original vector type if
8489 : supported. */
8490 1795 : unsigned lsize = n * GET_MODE_BITSIZE (elmode);
8491 1795 : unsigned int lnunits = const_nunits / n;
8492 : /* If we can't construct such a vector fall back to
8493 : element extracts from the original vector type and
8494 : element size stores. */
8495 1795 : if (int_mode_for_size (lsize, 0).exists (&elmode)
8496 1795 : && VECTOR_MODE_P (TYPE_MODE (vectype))
8497 1795 : && related_vector_mode (TYPE_MODE (vectype), elmode,
8498 1795 : lnunits).exists (&vmode)
8499 1767 : && (convert_optab_handler (vec_extract_optab,
8500 : vmode, elmode)
8501 : != CODE_FOR_nothing))
8502 : {
8503 1767 : nstores = lnunits;
8504 1767 : lnel = n;
8505 1767 : ltype = build_nonstandard_integer_type (lsize, 1);
8506 1767 : lvectype = build_vector_type (ltype, nstores);
8507 : }
8508 : /* Else fall back to vector extraction anyway.
8509 : Fewer stores are more important than avoiding spilling
8510 : of the vector we extract from. Compared to the
8511 : construction case in vectorizable_load no store-forwarding
8512 : issue exists here for reasonable archs. But only
8513 : if the store is supported. */
8514 28 : else if (!(dr_align == dr_aligned
8515 28 : || dr_align == dr_unaligned_supported))
8516 : {
8517 29452 : nstores = const_nunits;
8518 29452 : lnel = 1;
8519 29452 : ltype = elem_type;
8520 29452 : lvectype = vectype;
8521 : }
8522 : }
8523 : }
8524 :
8525 29452 : if (costing_p)
8526 : {
8527 : /* Record the decomposition type for target access during costing. */
8528 26092 : ls.ls_type = lvectype;
8529 26092 : ls.ls_eltype = ltype;
8530 : }
8531 : else
8532 3360 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
8533 :
8534 29452 : unsigned align;
8535 29452 : if (alignment_support_scheme == dr_aligned)
8536 1241 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
8537 : else
8538 28211 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
8539 : /* Alignment is at most the access size if we do multiple stores. */
8540 29452 : if (nstores > 1)
8541 26507 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
8542 29452 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
8543 29452 : int ncopies = vec_num;
8544 :
8545 29452 : if (!costing_p)
8546 : {
8547 3360 : ivstep = stride_step;
8548 :
8549 3360 : tree increment = fold_convert (TREE_TYPE (ivstep),
8550 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
8551 :
8552 3360 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
8553 : increment);
8554 :
8555 3360 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
8556 :
8557 3360 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
8558 3360 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
8559 3360 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
8560 : else
8561 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
8562 : true, NULL_TREE, true,
8563 : GSI_SAME_STMT);
8564 :
8565 3360 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
8566 : insert_after, &offvar, NULL,
8567 3360 : LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
8568 :
8569 3360 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
8570 : }
8571 :
8572 29452 : alias_off = build_int_cst (ref_type, 0);
8573 29452 : auto_vec<tree> vec_oprnds;
8574 : /* For costing some adjacent vector stores, we'd like to cost with
8575 : the total number of them once instead of cost each one by one. */
8576 29452 : unsigned int n_adjacent_stores = 0;
8577 29452 : running_off = offvar;
8578 29452 : if (!costing_p)
8579 3360 : vect_get_slp_defs (op_node, &vec_oprnds);
8580 29452 : unsigned int group_el = 0;
8581 29452 : unsigned HOST_WIDE_INT elsz
8582 29452 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
8583 70133 : for (j = 0; j < ncopies; j++)
8584 : {
8585 40681 : if (!costing_p)
8586 : {
8587 5196 : vec_oprnd = vec_oprnds[j];
8588 : /* Pun the vector to extract from if necessary. */
8589 5196 : if (lvectype != vectype)
8590 : {
8591 1008 : tree tem = make_ssa_name (lvectype);
8592 1008 : tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
8593 1008 : gimple *pun = gimple_build_assign (tem, cvt);
8594 1008 : vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
8595 1008 : vec_oprnd = tem;
8596 : }
8597 : }
8598 180641 : for (i = 0; i < nstores; i++)
8599 : {
8600 139960 : if (costing_p)
8601 : {
8602 124147 : n_adjacent_stores++;
8603 124147 : continue;
8604 : }
8605 15813 : tree newref, newoff;
8606 15813 : gimple *incr, *assign;
8607 15813 : tree size = TYPE_SIZE (ltype);
8608 : /* Extract the i'th component. */
8609 15813 : tree pos = fold_build2 (MULT_EXPR, bitsizetype,
8610 : bitsize_int (i), size);
8611 15813 : tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
8612 : size, pos);
8613 :
8614 15813 : elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
8615 : GSI_SAME_STMT);
8616 :
8617 15813 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
8618 15813 : group_el * elsz);
8619 15813 : newref = build2 (MEM_REF, ltype, running_off, this_off);
8620 15813 : vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
8621 :
8622 : /* And store it to *running_off. */
8623 15813 : assign = gimple_build_assign (newref, elem);
8624 15813 : vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
8625 :
8626 15813 : group_el += lnel;
8627 15813 : if (group_el == group_size)
8628 : {
8629 14176 : newoff = copy_ssa_name (running_off, NULL);
8630 14176 : incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
8631 : running_off, stride_step);
8632 14176 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
8633 :
8634 14176 : running_off = newoff;
8635 14176 : group_el = 0;
8636 : }
8637 : }
8638 : }
8639 :
8640 29452 : if (costing_p)
8641 : {
8642 26092 : if (n_adjacent_stores > 0)
8643 : {
8644 : /* Take a single lane vector type store as scalar
8645 : store to avoid ICE like 110776. */
8646 26092 : if (VECTOR_TYPE_P (ltype)
8647 26092 : && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
8648 1618 : vect_get_store_cost (vinfo, stmt_info, slp_node,
8649 : n_adjacent_stores, alignment_support_scheme,
8650 : misalignment, &inside_cost, cost_vec);
8651 : else
8652 24474 : inside_cost
8653 24474 : += record_stmt_cost (cost_vec, n_adjacent_stores,
8654 : scalar_store, slp_node, 0, vect_body);
8655 : /* Only need vector deconstruction when there is more
8656 : than one store. */
8657 26092 : if (nstores > 1)
8658 24058 : inside_cost
8659 24058 : += record_stmt_cost (cost_vec, ncopies,
8660 : vec_deconstruct, slp_node, 0, vect_body);
8661 : }
8662 26092 : if (dump_enabled_p ())
8663 658 : dump_printf_loc (MSG_NOTE, vect_location,
8664 : "vect_model_store_cost: inside_cost = %d, "
8665 : "prologue_cost = %d .\n",
8666 : inside_cost, prologue_cost);
8667 :
8668 26092 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8669 26092 : slp_node->data = new vect_load_store_data (std::move (ls));
8670 : }
8671 :
8672 29452 : return true;
8673 29452 : }
8674 :
8675 1356112 : gcc_assert (alignment_support_scheme);
8676 1356112 : vec_loop_masks *loop_masks
8677 197515 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
8678 1356112 : ? &LOOP_VINFO_MASKS (loop_vinfo)
8679 12 : : NULL);
8680 12 : vec_loop_lens *loop_lens
8681 197515 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
8682 : ? &LOOP_VINFO_LENS (loop_vinfo)
8683 0 : : NULL);
8684 :
8685 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8686 : are some difference here. We cannot enable both the lens and masks
8687 : during transform but it is allowed during analysis.
8688 : Shouldn't go with length-based approach if fully masked. */
8689 1356112 : if (cost_vec == NULL)
8690 : /* The cost_vec is NULL during transform. */
8691 552124 : gcc_assert ((!loop_lens || !loop_masks));
8692 :
8693 : /* Targets with store-lane instructions must not require explicit
8694 : realignment. vect_supportable_dr_alignment always returns either
8695 : dr_aligned or dr_unaligned_supported for masked operations. */
8696 1356112 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
8697 : && !mask_node
8698 : && !loop_masks)
8699 : || alignment_support_scheme == dr_aligned
8700 : || alignment_support_scheme == dr_unaligned_supported);
8701 :
8702 1356112 : tree offset = NULL_TREE;
8703 1356112 : if (!known_eq (poffset, 0))
8704 4647 : offset = size_int (poffset);
8705 :
8706 1356112 : tree dr_increment;
8707 1356112 : tree dr_bump;
8708 :
8709 1356112 : tree vec_offset = NULL_TREE;
8710 1356112 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8711 : {
8712 1476 : aggr_type = NULL_TREE;
8713 1476 : dr_increment = NULL_TREE;
8714 1476 : dr_bump = NULL_TREE;
8715 : }
8716 1354636 : else if (mat_gather_scatter_p (memory_access_type))
8717 : {
8718 0 : aggr_type = elem_type;
8719 0 : if (!costing_p)
8720 : {
8721 0 : tree vtype = ls.ls_type ? ls.ls_type : vectype;
8722 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
8723 : ls.strided_offset_vectype,
8724 : loop_vinfo, gsi,
8725 : &dr_increment, &dr_bump,
8726 : &vec_offset);
8727 : }
8728 : }
8729 : else
8730 : {
8731 1354636 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8732 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
8733 : else
8734 : aggr_type = vectype;
8735 1354636 : if (!costing_p)
8736 : {
8737 551649 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
8738 : memory_access_type);
8739 551649 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
8740 : memory_access_type);
8741 : }
8742 : }
8743 :
8744 1356112 : if (loop_vinfo && mask_node && !costing_p)
8745 527 : LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
8746 :
8747 : /* In case the vectorization factor (VF) is bigger than the number
8748 : of elements that we can fit in a vectype (nunits), we have to generate
8749 : more than one vector stmt - i.e - we need to "unroll" the
8750 : vector stmt by a factor VF/nunits. */
8751 :
8752 1356112 : auto_vec<tree> dr_chain (group_size);
8753 1356112 : auto_vec<tree> vec_masks;
8754 1356112 : tree vec_mask = NULL;
8755 1356112 : auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
8756 6111103 : for (i = 0; i < group_size; i++)
8757 3398879 : gvec_oprnds.quick_push (new auto_vec<tree> ());
8758 :
8759 1356112 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8760 : {
8761 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
8762 :
8763 0 : if (costing_p)
8764 : /* Update all incoming store operand nodes, the general handling
8765 : above only handles the mask and the first store operand node. */
8766 0 : for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
8767 0 : if (child != mask_node
8768 0 : && !vect_maybe_update_slp_op_vectype (child, vectype))
8769 : {
8770 0 : if (dump_enabled_p ())
8771 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8772 : "incompatible vector types for invariants\n");
8773 : return false;
8774 : }
8775 0 : unsigned inside_cost = 0, prologue_cost = 0;
8776 : /* For costing some adjacent vector stores, we'd like to cost with
8777 : the total number of them once instead of cost each one by one. */
8778 0 : unsigned int n_adjacent_stores = 0;
8779 0 : int ncopies = vec_num / group_size;
8780 0 : for (j = 0; j < ncopies; j++)
8781 : {
8782 0 : if (j == 0)
8783 : {
8784 0 : if (!costing_p)
8785 : {
8786 0 : if (mask_node)
8787 : {
8788 0 : vect_get_slp_defs (mask_node, &vec_masks);
8789 0 : vec_mask = vec_masks[0];
8790 : }
8791 0 : dataref_ptr
8792 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8793 : aggr_type, NULL, offset, &dummy,
8794 : gsi, NULL, false, dr_increment);
8795 : }
8796 : }
8797 0 : else if (!costing_p)
8798 : {
8799 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8800 0 : if (mask_node)
8801 0 : vec_mask = vec_masks[j];
8802 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
8803 : stmt_info, dr_bump);
8804 : }
8805 :
8806 0 : if (costing_p)
8807 : {
8808 0 : n_adjacent_stores += group_size;
8809 0 : continue;
8810 : }
8811 :
8812 : /* Get an array into which we can store the individual vectors. */
8813 0 : tree vec_array = create_vector_array (vectype, group_size);
8814 :
8815 : /* Invalidate the current contents of VEC_ARRAY. This should
8816 : become an RTL clobber too, which prevents the vector registers
8817 : from being upward-exposed. */
8818 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8819 :
8820 : /* Store the individual vectors into the array. */
8821 0 : for (i = 0; i < group_size; i++)
8822 : {
8823 0 : slp_tree child;
8824 0 : if (i == 0 || !mask_node)
8825 0 : child = SLP_TREE_CHILDREN (slp_node)[i];
8826 : else
8827 0 : child = SLP_TREE_CHILDREN (slp_node)[i + 1];
8828 0 : vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
8829 0 : write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
8830 : i);
8831 : }
8832 :
8833 0 : tree final_mask = NULL;
8834 0 : tree final_len = NULL;
8835 0 : tree bias = NULL;
8836 0 : if (loop_masks)
8837 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
8838 : ncopies, vectype, j);
8839 0 : if (vec_mask)
8840 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
8841 : vec_mask, gsi);
8842 :
8843 0 : if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
8844 : {
8845 0 : if (loop_lens)
8846 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
8847 : ncopies, vectype, j, 1, true);
8848 : else
8849 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8850 0 : signed char biasval
8851 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
8852 0 : bias = build_int_cst (intQI_type_node, biasval);
8853 0 : if (!final_mask)
8854 : {
8855 0 : mask_vectype = truth_type_for (vectype);
8856 0 : final_mask = build_minus_one_cst (mask_vectype);
8857 : }
8858 : }
8859 :
8860 0 : gcall *call;
8861 0 : if (final_len && final_mask)
8862 : {
8863 : /* Emit:
8864 : MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8865 : LEN, BIAS, VEC_ARRAY). */
8866 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8867 0 : tree alias_ptr = build_int_cst (ref_type, align);
8868 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
8869 : dataref_ptr, alias_ptr,
8870 : final_mask, final_len, bias,
8871 : vec_array);
8872 : }
8873 0 : else if (final_mask)
8874 : {
8875 : /* Emit:
8876 : MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8877 : VEC_ARRAY). */
8878 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8879 0 : tree alias_ptr = build_int_cst (ref_type, align);
8880 0 : call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
8881 : dataref_ptr, alias_ptr,
8882 : final_mask, vec_array);
8883 : }
8884 : else
8885 : {
8886 : /* Emit:
8887 : MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8888 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8889 0 : call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
8890 0 : gimple_call_set_lhs (call, data_ref);
8891 : }
8892 0 : gimple_call_set_nothrow (call, true);
8893 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
8894 :
8895 : /* Record that VEC_ARRAY is now dead. */
8896 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8897 : }
8898 :
8899 0 : if (costing_p)
8900 : {
8901 0 : if (n_adjacent_stores > 0)
8902 0 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
8903 : alignment_support_scheme, misalignment,
8904 : &inside_cost, cost_vec);
8905 0 : if (dump_enabled_p ())
8906 0 : dump_printf_loc (MSG_NOTE, vect_location,
8907 : "vect_model_store_cost: inside_cost = %d, "
8908 : "prologue_cost = %d .\n",
8909 : inside_cost, prologue_cost);
8910 :
8911 0 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8912 0 : slp_node->data = new vect_load_store_data (std::move (ls));
8913 : }
8914 :
8915 : return true;
8916 : }
8917 :
8918 1356112 : if (mat_gather_scatter_p (memory_access_type))
8919 : {
8920 1476 : gcc_assert (!grouped_store || ls.ls_type);
8921 1476 : if (ls.ls_type)
8922 0 : vectype = ls.ls_type;
8923 1476 : auto_vec<tree> vec_offsets;
8924 1476 : unsigned int inside_cost = 0, prologue_cost = 0;
8925 1476 : int num_stmts = vec_num;
8926 3351 : for (j = 0; j < num_stmts; j++)
8927 : {
8928 1875 : gimple *new_stmt;
8929 1875 : if (j == 0)
8930 : {
8931 1476 : if (costing_p && vls_type == VLS_STORE_INVARIANT)
8932 218 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8933 : slp_node, 0, vect_prologue);
8934 : else if (!costing_p)
8935 : {
8936 : /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8937 : DR_CHAIN is of size 1. */
8938 475 : gcc_assert (group_size == 1);
8939 475 : vect_get_slp_defs (op_node, gvec_oprnds[0]);
8940 475 : if (mask_node)
8941 70 : vect_get_slp_defs (mask_node, &vec_masks);
8942 :
8943 475 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8944 475 : vect_get_gather_scatter_ops (loop, slp_node,
8945 : &dataref_ptr, &vec_offsets);
8946 : else
8947 0 : dataref_ptr
8948 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8949 : aggr_type, NULL, offset,
8950 : &dummy, gsi, NULL, false,
8951 : dr_increment);
8952 : }
8953 : }
8954 399 : else if (!costing_p)
8955 : {
8956 35 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8957 35 : if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8958 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
8959 : gsi, stmt_info, dr_bump);
8960 : }
8961 :
8962 2603 : new_stmt = NULL;
8963 728 : if (!costing_p)
8964 : {
8965 510 : vec_oprnd = (*gvec_oprnds[0])[j];
8966 510 : if (mask_node)
8967 90 : vec_mask = vec_masks[j];
8968 : /* We should have caught mismatched types earlier. */
8969 510 : gcc_assert (ls.ls_type
8970 : || useless_type_conversion_p
8971 : (vectype, TREE_TYPE (vec_oprnd)));
8972 : }
8973 510 : tree final_mask = NULL_TREE;
8974 2385 : tree final_len = NULL_TREE;
8975 2385 : tree bias = NULL_TREE;
8976 510 : if (!costing_p)
8977 : {
8978 510 : if (loop_masks)
8979 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi,
8980 : loop_masks, num_stmts,
8981 : vectype, j);
8982 510 : if (vec_mask)
8983 90 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
8984 : final_mask, vec_mask, gsi);
8985 : }
8986 :
8987 1875 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
8988 1875 : tree alias_align_ptr = build_int_cst (ref_type, align);
8989 1875 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
8990 : {
8991 0 : if (costing_p)
8992 : {
8993 0 : if (ls.supported_offset_vectype
8994 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
8995 : vec_offset))
8996 0 : inside_cost
8997 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
8998 : slp_node, 0, vect_body);
8999 0 : if (ls.supported_scale)
9000 0 : inside_cost
9001 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9002 : slp_node, 0, vect_body);
9003 :
9004 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9005 0 : inside_cost
9006 0 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9007 : slp_node, 0, vect_body);
9008 0 : continue;
9009 0 : }
9010 :
9011 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9012 0 : vec_offset = vec_offsets[j];
9013 :
9014 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9015 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
9016 :
9017 : /* Perform the offset conversion and scaling if necessary. */
9018 0 : if (!strided
9019 0 : && (ls.supported_offset_vectype || ls.supported_scale))
9020 : {
9021 0 : gimple_seq stmts = NULL;
9022 0 : if (ls.supported_offset_vectype)
9023 0 : vec_offset = gimple_convert
9024 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
9025 0 : if (ls.supported_scale)
9026 : {
9027 : /* Only scale the vec_offset if we haven't already. */
9028 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
9029 0 : || j == 0)
9030 : {
9031 0 : tree mult_cst = build_int_cst
9032 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
9033 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
9034 0 : tree mult = build_vector_from_val
9035 0 : (TREE_TYPE (vec_offset), mult_cst);
9036 0 : vec_offset = gimple_build
9037 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
9038 : vec_offset, mult);
9039 : }
9040 0 : scale = size_int (ls.supported_scale);
9041 : }
9042 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9043 : }
9044 :
9045 0 : if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
9046 : {
9047 0 : if (loop_lens)
9048 0 : final_len = vect_get_loop_len (loop_vinfo, gsi,
9049 : loop_lens, num_stmts,
9050 : vectype, j, 1, true);
9051 : else
9052 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9053 :
9054 0 : signed char biasval
9055 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9056 0 : bias = build_int_cst (intQI_type_node, biasval);
9057 0 : if (!final_mask)
9058 : {
9059 0 : mask_vectype = truth_type_for (vectype);
9060 0 : final_mask = build_minus_one_cst (mask_vectype);
9061 : }
9062 : }
9063 :
9064 0 : if (ls.ls_type)
9065 : {
9066 0 : gimple *conv_stmt
9067 0 : = gimple_build_assign (make_ssa_name (vectype),
9068 : VIEW_CONVERT_EXPR,
9069 : build1 (VIEW_CONVERT_EXPR, vectype,
9070 : vec_oprnd));
9071 0 : vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
9072 : gsi);
9073 0 : vec_oprnd = gimple_get_lhs (conv_stmt);
9074 : }
9075 :
9076 0 : gcall *call;
9077 0 : if (final_len && final_mask)
9078 : {
9079 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
9080 0 : call = gimple_build_call_internal (
9081 : IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
9082 : alias_align_ptr,
9083 : vec_offset, scale, vec_oprnd, final_mask, final_len,
9084 : bias);
9085 : else
9086 : /* Non-vector offset indicates that prefer to take
9087 : MASK_LEN_STRIDED_STORE instead of the
9088 : IFN_MASK_SCATTER_STORE with direct stride arg.
9089 : Similar to the gather case we have checked the
9090 : alignment for a scatter already and assume
9091 : that the strided store has the same requirements. */
9092 0 : call = gimple_build_call_internal (
9093 : IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
9094 : vec_offset, vec_oprnd, final_mask, final_len, bias);
9095 : }
9096 0 : else if (final_mask)
9097 0 : call = gimple_build_call_internal
9098 0 : (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
9099 : alias_align_ptr,
9100 : vec_offset, scale, vec_oprnd, final_mask);
9101 : else
9102 0 : call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
9103 : dataref_ptr,
9104 : alias_align_ptr,
9105 : vec_offset,
9106 : scale, vec_oprnd);
9107 0 : gimple_call_set_nothrow (call, true);
9108 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9109 0 : new_stmt = call;
9110 : }
9111 1875 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
9112 : {
9113 : /* The builtin decls path for scatter is legacy, x86 only. */
9114 331 : gcc_assert (nunits.is_constant ()
9115 : && (!final_mask
9116 : || SCALAR_INT_MODE_P
9117 : (TYPE_MODE (TREE_TYPE (final_mask)))));
9118 331 : if (costing_p)
9119 : {
9120 200 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9121 200 : inside_cost
9122 200 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9123 : slp_node, 0, vect_body);
9124 200 : continue;
9125 200 : }
9126 :
9127 131 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9128 131 : poly_uint64 offset_nunits
9129 131 : = TYPE_VECTOR_SUBPARTS (offset_vectype);
9130 131 : if (known_eq (nunits, offset_nunits))
9131 : {
9132 55 : new_stmt = vect_build_one_scatter_store_call
9133 110 : (vinfo, stmt_info, slp_node, gsi,
9134 55 : ls.gs.decl, dataref_ptr, vec_offsets[j],
9135 : vec_oprnd, final_mask);
9136 55 : vect_finish_stmt_generation (vinfo, stmt_info,
9137 : new_stmt, gsi);
9138 : }
9139 76 : else if (known_eq (nunits, offset_nunits * 2))
9140 : {
9141 : /* We have a offset vector with half the number of
9142 : lanes but the builtins will store full vectype
9143 : data from the lower lanes. */
9144 30 : new_stmt = vect_build_one_scatter_store_call
9145 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9146 30 : dataref_ptr, vec_offsets[2 * j],
9147 : vec_oprnd, final_mask);
9148 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9149 : new_stmt, gsi);
9150 30 : int count = nunits.to_constant ();
9151 30 : vec_perm_builder sel (count, count, 1);
9152 30 : sel.quick_grow (count);
9153 412 : for (int i = 0; i < count; ++i)
9154 352 : sel[i] = i | (count / 2);
9155 30 : vec_perm_indices indices (sel, 2, count);
9156 30 : tree perm_mask
9157 30 : = vect_gen_perm_mask_checked (vectype, indices);
9158 30 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
9159 : vec_oprnd, vec_oprnd,
9160 : perm_mask);
9161 30 : vec_oprnd = make_ssa_name (vectype);
9162 30 : gimple_set_lhs (new_stmt, vec_oprnd);
9163 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9164 : new_stmt, gsi);
9165 30 : if (final_mask)
9166 : {
9167 20 : new_stmt = gimple_build_assign (NULL_TREE,
9168 : VEC_UNPACK_HI_EXPR,
9169 : final_mask);
9170 20 : final_mask = make_ssa_name
9171 20 : (truth_type_for (offset_vectype));
9172 20 : gimple_set_lhs (new_stmt, final_mask);
9173 20 : vect_finish_stmt_generation (vinfo, stmt_info,
9174 : new_stmt, gsi);
9175 : }
9176 :
9177 30 : new_stmt = vect_build_one_scatter_store_call
9178 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9179 30 : dataref_ptr, vec_offsets[2 * j + 1],
9180 : vec_oprnd, final_mask);
9181 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9182 : new_stmt, gsi);
9183 30 : }
9184 46 : else if (known_eq (nunits * 2, offset_nunits))
9185 : {
9186 : /* We have a offset vector with double the number of
9187 : lanes. Select the low/high part accordingly. */
9188 46 : vec_offset = vec_offsets[j / 2];
9189 46 : if (j & 1)
9190 : {
9191 23 : int count = offset_nunits.to_constant ();
9192 23 : vec_perm_builder sel (count, count, 1);
9193 23 : sel.quick_grow (count);
9194 286 : for (int i = 0; i < count; ++i)
9195 240 : sel[i] = i | (count / 2);
9196 23 : vec_perm_indices indices (sel, 2, count);
9197 23 : tree perm_mask = vect_gen_perm_mask_checked
9198 23 : (TREE_TYPE (vec_offset), indices);
9199 23 : new_stmt = gimple_build_assign (NULL_TREE,
9200 : VEC_PERM_EXPR,
9201 : vec_offset,
9202 : vec_offset,
9203 : perm_mask);
9204 23 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
9205 23 : gimple_set_lhs (new_stmt, vec_offset);
9206 23 : vect_finish_stmt_generation (vinfo, stmt_info,
9207 : new_stmt, gsi);
9208 23 : }
9209 :
9210 46 : new_stmt = vect_build_one_scatter_store_call
9211 46 : (vinfo, stmt_info, slp_node, gsi,
9212 : ls.gs.decl, dataref_ptr, vec_offset,
9213 : vec_oprnd, final_mask);
9214 46 : vect_finish_stmt_generation (vinfo, stmt_info,
9215 : new_stmt, gsi);
9216 : }
9217 : else
9218 0 : gcc_unreachable ();
9219 : }
9220 : else
9221 : {
9222 : /* Emulated scatter. */
9223 1544 : gcc_assert (!final_mask);
9224 1544 : if (costing_p)
9225 : {
9226 1165 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9227 : /* For emulated scatter N offset vector element extracts
9228 : (we assume the scalar scaling and ptr + offset add is
9229 : consumed by the load). */
9230 1165 : inside_cost
9231 1165 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9232 : slp_node, 0, vect_body);
9233 : /* N scalar stores plus extracting the elements. */
9234 1165 : inside_cost
9235 1165 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9236 : slp_node, 0, vect_body);
9237 1165 : inside_cost
9238 1165 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9239 : slp_node, 0, vect_body);
9240 1165 : continue;
9241 1165 : }
9242 :
9243 379 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9244 379 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
9245 379 : unsigned HOST_WIDE_INT const_offset_nunits
9246 379 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
9247 379 : vec<constructor_elt, va_gc> *ctor_elts;
9248 379 : vec_alloc (ctor_elts, const_nunits);
9249 379 : gimple_seq stmts = NULL;
9250 379 : tree elt_type = TREE_TYPE (vectype);
9251 379 : unsigned HOST_WIDE_INT elt_size
9252 379 : = tree_to_uhwi (TYPE_SIZE (elt_type));
9253 : /* We support offset vectors with more elements
9254 : than the data vector for now. */
9255 379 : unsigned HOST_WIDE_INT factor
9256 : = const_offset_nunits / const_nunits;
9257 379 : vec_offset = vec_offsets[j / factor];
9258 379 : unsigned elt_offset
9259 379 : = (j % factor) * const_nunits;
9260 379 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
9261 379 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9262 379 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
9263 1922 : for (unsigned k = 0; k < const_nunits; ++k)
9264 : {
9265 : /* Compute the offsetted pointer. */
9266 1164 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
9267 : bitsize_int (k + elt_offset));
9268 1164 : tree idx
9269 2328 : = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
9270 1164 : vec_offset, TYPE_SIZE (idx_type), boff);
9271 1164 : idx = gimple_convert (&stmts, sizetype, idx);
9272 1164 : idx = gimple_build (&stmts, MULT_EXPR, sizetype,
9273 : idx, scale);
9274 1164 : tree ptr
9275 1164 : = gimple_build (&stmts, PLUS_EXPR,
9276 1164 : TREE_TYPE (dataref_ptr),
9277 : dataref_ptr, idx);
9278 1164 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
9279 : /* Extract the element to be stored. */
9280 1164 : tree elt
9281 2328 : = gimple_build (&stmts, BIT_FIELD_REF,
9282 1164 : TREE_TYPE (vectype),
9283 1164 : vec_oprnd, TYPE_SIZE (elt_type),
9284 1164 : bitsize_int (k * elt_size));
9285 1164 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9286 1164 : stmts = NULL;
9287 1164 : tree ref
9288 1164 : = build2 (MEM_REF, ltype, ptr,
9289 : build_int_cst (ref_type, 0));
9290 1164 : new_stmt = gimple_build_assign (ref, elt);
9291 1164 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9292 : }
9293 :
9294 379 : slp_node->push_vec_def (new_stmt);
9295 : }
9296 : }
9297 :
9298 1476 : if (costing_p)
9299 : {
9300 1001 : if (dump_enabled_p ())
9301 78 : dump_printf_loc (MSG_NOTE, vect_location,
9302 : "vect_model_store_cost: inside_cost = %d, "
9303 : "prologue_cost = %d .\n",
9304 : inside_cost, prologue_cost);
9305 1001 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9306 1001 : slp_node->data = new vect_load_store_data (std::move (ls));
9307 : }
9308 :
9309 1476 : return true;
9310 1476 : }
9311 :
9312 1354636 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
9313 : || memory_access_type == VMAT_CONTIGUOUS_DOWN
9314 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
9315 :
9316 1354636 : unsigned inside_cost = 0, prologue_cost = 0;
9317 : /* For costing some adjacent vector stores, we'd like to cost with
9318 : the total number of them once instead of cost each one by one. */
9319 1354636 : unsigned int n_adjacent_stores = 0;
9320 1354636 : auto_vec<tree> result_chain (group_size);
9321 1354636 : auto_vec<tree, 1> vec_oprnds;
9322 1354636 : gimple *new_stmt;
9323 1354636 : if (!costing_p)
9324 : {
9325 : /* Get vectorized arguments for SLP_NODE. */
9326 551649 : vect_get_slp_defs (op_node, &vec_oprnds);
9327 551649 : vec_oprnd = vec_oprnds[0];
9328 551649 : if (mask_node)
9329 : {
9330 457 : vect_get_slp_defs (mask_node, &vec_masks);
9331 457 : vec_mask = vec_masks[0];
9332 : }
9333 : }
9334 :
9335 : /* We should have caught mismatched types earlier. */
9336 551649 : gcc_assert (costing_p
9337 : || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
9338 1354636 : bool simd_lane_access_p
9339 1354636 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
9340 1354636 : if (!costing_p
9341 1354636 : && simd_lane_access_p
9342 4358 : && !loop_masks
9343 4358 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
9344 4358 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
9345 4358 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
9346 4358 : && integer_zerop (DR_INIT (first_dr_info->dr))
9347 1358994 : && alias_sets_conflict_p (get_alias_set (aggr_type),
9348 4358 : get_alias_set (TREE_TYPE (ref_type))))
9349 : {
9350 4350 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
9351 4350 : dataref_offset = build_int_cst (ref_type, 0);
9352 : }
9353 1350286 : else if (!costing_p)
9354 1094590 : dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
9355 : simd_lane_access_p ? loop : NULL,
9356 : offset, &dummy, gsi, NULL,
9357 : simd_lane_access_p, dr_increment);
9358 :
9359 1354636 : new_stmt = NULL;
9360 1354636 : gcc_assert (!grouped_store);
9361 3011723 : for (i = 0; i < vec_num; i++)
9362 : {
9363 1657087 : if (!costing_p)
9364 683242 : vec_oprnd = vec_oprnds[i];
9365 :
9366 1657087 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
9367 : {
9368 3327 : if (costing_p)
9369 2189 : inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
9370 : slp_node, 0, vect_body);
9371 : else
9372 : {
9373 1138 : tree perm_mask = perm_mask_for_reverse (vectype);
9374 1138 : tree new_temp = make_ssa_name (vectype);
9375 :
9376 : /* Generate the permute statement. */
9377 1138 : gimple *perm_stmt
9378 1138 : = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
9379 : vec_oprnd, perm_mask);
9380 1138 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9381 :
9382 1138 : perm_stmt = SSA_NAME_DEF_STMT (new_temp);
9383 1657087 : vec_oprnd = new_temp;
9384 : }
9385 : }
9386 :
9387 1657087 : if (costing_p)
9388 : {
9389 973845 : n_adjacent_stores++;
9390 973845 : continue;
9391 : }
9392 :
9393 683242 : tree final_mask = NULL_TREE;
9394 683242 : tree final_len = NULL_TREE;
9395 683242 : tree bias = NULL_TREE;
9396 683242 : if (loop_masks)
9397 78 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
9398 : vec_num, vectype, i);
9399 683242 : if (vec_mask)
9400 658 : vec_mask = vec_masks[i];
9401 658 : if (vec_mask)
9402 658 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
9403 : vec_mask, gsi);
9404 :
9405 683242 : if (i > 0)
9406 : /* Bump the vector pointer. */
9407 131593 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
9408 : dr_bump);
9409 :
9410 683242 : unsigned misalign;
9411 683242 : unsigned HOST_WIDE_INT align;
9412 683242 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
9413 683242 : if (alignment_support_scheme == dr_aligned)
9414 : misalign = 0;
9415 312428 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
9416 : {
9417 163047 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
9418 163047 : misalign = 0;
9419 : }
9420 : else
9421 149381 : misalign = misalignment;
9422 683242 : if (dataref_offset == NULL_TREE
9423 677883 : && TREE_CODE (dataref_ptr) == SSA_NAME)
9424 185307 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, misalign);
9425 683242 : align = least_bit_hwi (misalign | align);
9426 :
9427 : /* Compute IFN when LOOP_LENS or final_mask valid. */
9428 683242 : machine_mode vmode = TYPE_MODE (vectype);
9429 683242 : machine_mode new_vmode = vmode;
9430 683242 : internal_fn partial_ifn = IFN_LAST;
9431 683242 : if (loop_lens)
9432 : {
9433 0 : opt_machine_mode new_ovmode
9434 0 : = get_len_load_store_mode (vmode, false, &partial_ifn);
9435 0 : new_vmode = new_ovmode.require ();
9436 0 : unsigned factor
9437 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
9438 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
9439 : vec_num, vectype, i, factor, true);
9440 : }
9441 683242 : else if (final_mask)
9442 : {
9443 671 : if (!can_vec_mask_load_store_p (vmode,
9444 671 : TYPE_MODE (TREE_TYPE (final_mask)),
9445 : false, &partial_ifn))
9446 0 : gcc_unreachable ();
9447 : }
9448 :
9449 683242 : if (partial_ifn == IFN_MASK_LEN_STORE)
9450 : {
9451 0 : if (!final_len)
9452 : {
9453 : /* Pass VF value to 'len' argument of
9454 : MASK_LEN_STORE if LOOP_LENS is invalid. */
9455 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9456 : }
9457 0 : if (!final_mask)
9458 : {
9459 : /* Pass all ones value to 'mask' argument of
9460 : MASK_LEN_STORE if final_mask is invalid. */
9461 0 : mask_vectype = truth_type_for (vectype);
9462 0 : final_mask = build_minus_one_cst (mask_vectype);
9463 : }
9464 : }
9465 683242 : if (final_len)
9466 : {
9467 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9468 0 : bias = build_int_cst (intQI_type_node, biasval);
9469 : }
9470 :
9471 : /* Arguments are ready. Create the new vector stmt. */
9472 0 : if (final_len)
9473 : {
9474 0 : gcall *call;
9475 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9476 : /* Need conversion if it's wrapped with VnQI. */
9477 0 : if (vmode != new_vmode)
9478 : {
9479 0 : tree new_vtype
9480 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
9481 : new_vmode);
9482 0 : tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
9483 0 : vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
9484 0 : gassign *new_stmt
9485 0 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
9486 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9487 0 : vec_oprnd = var;
9488 : }
9489 :
9490 0 : if (partial_ifn == IFN_MASK_LEN_STORE)
9491 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
9492 : dataref_ptr, ptr, final_mask,
9493 : final_len, bias, vec_oprnd);
9494 : else
9495 0 : call = gimple_build_call_internal (IFN_LEN_STORE, 5,
9496 : dataref_ptr, ptr, final_len,
9497 : bias, vec_oprnd);
9498 0 : gimple_call_set_nothrow (call, true);
9499 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9500 0 : new_stmt = call;
9501 : }
9502 683242 : else if (final_mask)
9503 : {
9504 671 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9505 671 : gcall *call
9506 671 : = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
9507 : ptr, final_mask, vec_oprnd);
9508 671 : gimple_call_set_nothrow (call, true);
9509 671 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9510 671 : new_stmt = call;
9511 : }
9512 : else
9513 : {
9514 682571 : data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
9515 : dataref_offset ? dataref_offset
9516 : : build_int_cst (ref_type, 0));
9517 682571 : if (alignment_support_scheme == dr_aligned
9518 682571 : && align >= TYPE_ALIGN_UNIT (vectype))
9519 : ;
9520 : else
9521 311915 : TREE_TYPE (data_ref)
9522 623830 : = build_aligned_type (TREE_TYPE (data_ref),
9523 : align * BITS_PER_UNIT);
9524 682571 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
9525 682571 : new_stmt = gimple_build_assign (data_ref, vec_oprnd);
9526 682571 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9527 : }
9528 : }
9529 :
9530 1354636 : if (costing_p)
9531 : {
9532 802987 : if (n_adjacent_stores > 0)
9533 802987 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
9534 : alignment_support_scheme, misalignment,
9535 : &inside_cost, cost_vec);
9536 :
9537 : /* When vectorizing a store into the function result assign
9538 : a penalty if the function returns in a multi-register location.
9539 : In this case we assume we'll end up with having to spill the
9540 : vector result and do piecewise loads as a conservative estimate. */
9541 802987 : tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
9542 802987 : if (base
9543 802987 : && (TREE_CODE (base) == RESULT_DECL
9544 752598 : || (DECL_P (base) && cfun_returns (base)))
9545 865226 : && !aggregate_value_p (base, cfun->decl))
9546 : {
9547 11151 : rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
9548 : /* ??? Handle PARALLEL in some way. */
9549 11151 : if (REG_P (reg))
9550 : {
9551 10943 : int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
9552 : /* Assume that a single reg-reg move is possible and cheap,
9553 : do not account for vector to gp register move cost. */
9554 10943 : if (nregs > 1)
9555 : {
9556 : /* Spill. */
9557 10075 : prologue_cost
9558 10075 : += record_stmt_cost (cost_vec, 1, vector_store,
9559 : slp_node, 0, vect_epilogue);
9560 : /* Loads. */
9561 10075 : prologue_cost
9562 10075 : += record_stmt_cost (cost_vec, nregs, scalar_load,
9563 : slp_node, 0, vect_epilogue);
9564 : }
9565 : }
9566 : }
9567 802987 : if (dump_enabled_p ())
9568 13910 : dump_printf_loc (MSG_NOTE, vect_location,
9569 : "vect_model_store_cost: inside_cost = %d, "
9570 : "prologue_cost = %d .\n",
9571 : inside_cost, prologue_cost);
9572 :
9573 802987 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9574 802987 : slp_node->data = new vect_load_store_data (std::move (ls));
9575 : }
9576 :
9577 1354636 : return true;
9578 1387406 : }
9579 :
9580 : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9581 : VECTOR_CST mask. No checks are made that the target platform supports the
9582 : mask, so callers may wish to test can_vec_perm_const_p separately, or use
9583 : vect_gen_perm_mask_checked. */
9584 :
9585 : tree
9586 62454 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
9587 : {
9588 62454 : tree mask_type;
9589 :
9590 62454 : poly_uint64 nunits = sel.length ();
9591 62454 : gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
9592 :
9593 62454 : mask_type = build_vector_type (ssizetype, nunits);
9594 62454 : return vec_perm_indices_to_tree (mask_type, sel);
9595 : }
9596 :
9597 : /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9598 : i.e. that the target supports the pattern _for arbitrary input vectors_. */
9599 :
9600 : tree
9601 59596 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
9602 : {
9603 59596 : machine_mode vmode = TYPE_MODE (vectype);
9604 59596 : gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
9605 59596 : return vect_gen_perm_mask_any (vectype, sel);
9606 : }
9607 :
9608 : /* Given a vector variable X and Y, that was generated for the scalar
9609 : STMT_INFO, generate instructions to permute the vector elements of X and Y
9610 : using permutation mask MASK_VEC, insert them at *GSI and return the
9611 : permuted vector variable. */
9612 :
9613 : static tree
9614 1442 : permute_vec_elements (vec_info *vinfo,
9615 : tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
9616 : gimple_stmt_iterator *gsi)
9617 : {
9618 1442 : tree vectype = TREE_TYPE (x);
9619 1442 : tree perm_dest, data_ref;
9620 1442 : gimple *perm_stmt;
9621 :
9622 1442 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
9623 1442 : if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
9624 1442 : perm_dest = vect_create_destination_var (scalar_dest, vectype);
9625 : else
9626 0 : perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
9627 1442 : data_ref = make_ssa_name (perm_dest);
9628 :
9629 : /* Generate the permute statement. */
9630 1442 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
9631 1442 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9632 :
9633 1442 : return data_ref;
9634 : }
9635 :
9636 : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9637 : inserting them on the loops preheader edge. Returns true if we
9638 : were successful in doing so (and thus STMT_INFO can be moved then),
9639 : otherwise returns false. HOIST_P indicates if we want to hoist the
9640 : definitions of all SSA uses, it would be false when we are costing. */
9641 :
9642 : static bool
9643 4006 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
9644 : {
9645 4006 : ssa_op_iter i;
9646 4006 : use_operand_p use_p;
9647 4006 : auto_vec<use_operand_p, 8> to_hoist;
9648 :
9649 7579 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
9650 : {
9651 3603 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9652 3603 : if (!gimple_nop_p (def_stmt)
9653 3603 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
9654 : {
9655 : /* Make sure we don't need to recurse. While we could do
9656 : so in simple cases when there are more complex use webs
9657 : we don't have an easy way to preserve stmt order to fulfil
9658 : dependencies within them. */
9659 93 : tree op2;
9660 93 : ssa_op_iter i2;
9661 93 : if (gimple_code (def_stmt) == GIMPLE_PHI
9662 93 : || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
9663 : == NULL_DEF_OPERAND_P))
9664 30 : return false;
9665 188 : FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
9666 : {
9667 125 : gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
9668 125 : if (!gimple_nop_p (def_stmt2)
9669 125 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
9670 : return false;
9671 : }
9672 63 : to_hoist.safe_push (use_p);
9673 : }
9674 : }
9675 :
9676 7982 : if (to_hoist.is_empty ())
9677 : return true;
9678 :
9679 39 : if (!hoist_p)
9680 : return true;
9681 :
9682 : /* Instead of moving defs we copy them so we can zero their UID to not
9683 : confuse dominance queries in the preheader. */
9684 5 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
9685 20 : for (use_operand_p use_p : to_hoist)
9686 : {
9687 5 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9688 5 : gimple *copy = gimple_copy (def_stmt);
9689 5 : gimple_set_uid (copy, 0);
9690 5 : def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
9691 5 : tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
9692 5 : update_stmt (copy);
9693 5 : def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
9694 5 : SET_DEF (def_p, new_def);
9695 5 : SET_USE (use_p, new_def);
9696 5 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
9697 : }
9698 :
9699 : return true;
9700 4006 : }
9701 :
9702 : /* vectorizable_load.
9703 :
9704 : Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9705 : that can be vectorized.
9706 : If COST_VEC is passed, calculate costs but don't change anything,
9707 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
9708 : it, and insert it at GSI.
9709 : Return true if STMT_INFO is vectorizable in this way. */
9710 :
9711 : static bool
9712 2219718 : vectorizable_load (vec_info *vinfo,
9713 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
9714 : slp_tree slp_node,
9715 : stmt_vector_for_cost *cost_vec)
9716 : {
9717 2219718 : tree scalar_dest;
9718 2219718 : tree vec_dest = NULL;
9719 2219718 : tree data_ref = NULL;
9720 2219718 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
9721 2219718 : class loop *loop = NULL;
9722 2219718 : class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
9723 2219718 : bool nested_in_vect_loop = false;
9724 2219718 : tree elem_type;
9725 : /* Avoid false positive uninitialized warning, see PR110652. */
9726 2219718 : tree new_temp = NULL_TREE;
9727 2219718 : machine_mode mode;
9728 2219718 : tree dummy;
9729 2219718 : tree dataref_ptr = NULL_TREE;
9730 2219718 : tree dataref_offset = NULL_TREE;
9731 2219718 : int i, j;
9732 2219718 : unsigned int group_size;
9733 2219718 : poly_uint64 group_gap_adj;
9734 2219718 : tree msq = NULL_TREE, lsq;
9735 2219718 : tree realignment_token = NULL_TREE;
9736 2219718 : gphi *phi = NULL;
9737 2219718 : bool grouped_load = false;
9738 2219718 : stmt_vec_info first_stmt_info;
9739 2219718 : stmt_vec_info first_stmt_info_for_drptr = NULL;
9740 2219718 : bool compute_in_loop = false;
9741 2219718 : class loop *at_loop;
9742 2219718 : int vec_num;
9743 2219718 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
9744 2219718 : poly_uint64 vf;
9745 2219718 : tree aggr_type;
9746 2219718 : tree ref_type;
9747 2219718 : enum vect_def_type mask_dt = vect_unknown_def_type;
9748 2219718 : enum vect_def_type els_dt = vect_unknown_def_type;
9749 :
9750 2219718 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
9751 : return false;
9752 :
9753 2219718 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
9754 244447 : && cost_vec)
9755 : return false;
9756 :
9757 1975271 : if (!STMT_VINFO_DATA_REF (stmt_info))
9758 : return false;
9759 :
9760 1572792 : tree mask_vectype = NULL_TREE;
9761 1572792 : tree els = NULL_TREE; tree els_vectype = NULL_TREE;
9762 :
9763 1572792 : int mask_index = -1;
9764 1572792 : int els_index = -1;
9765 1572792 : slp_tree mask_node = NULL;
9766 1572792 : slp_tree els_op = NULL;
9767 1572792 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
9768 : {
9769 1568372 : scalar_dest = gimple_assign_lhs (assign);
9770 1568372 : if (TREE_CODE (scalar_dest) != SSA_NAME)
9771 : return false;
9772 :
9773 738260 : tree_code code = gimple_assign_rhs_code (assign);
9774 738260 : if (code != ARRAY_REF
9775 738260 : && code != BIT_FIELD_REF
9776 738260 : && code != INDIRECT_REF
9777 513694 : && code != COMPONENT_REF
9778 513694 : && code != IMAGPART_EXPR
9779 366477 : && code != REALPART_EXPR
9780 366477 : && code != MEM_REF
9781 294 : && TREE_CODE_CLASS (code) != tcc_declaration)
9782 : return false;
9783 : }
9784 : else
9785 : {
9786 4420 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
9787 4420 : if (!call || !gimple_call_internal_p (call))
9788 : return false;
9789 :
9790 4420 : internal_fn ifn = gimple_call_internal_fn (call);
9791 4420 : if (!internal_load_fn_p (ifn))
9792 : return false;
9793 :
9794 3090 : scalar_dest = gimple_call_lhs (call);
9795 3090 : if (!scalar_dest)
9796 : return false;
9797 :
9798 3090 : mask_index = internal_fn_mask_index (ifn);
9799 3090 : if (mask_index >= 0)
9800 3090 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
9801 3090 : if (mask_index >= 0
9802 3090 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
9803 : &mask_node, &mask_dt, &mask_vectype))
9804 : return false;
9805 :
9806 3090 : els_index = internal_fn_else_index (ifn);
9807 3090 : if (els_index >= 0)
9808 3090 : els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
9809 3090 : if (els_index >= 0
9810 3090 : && !vect_is_simple_use (vinfo, slp_node, els_index,
9811 : &els, &els_op, &els_dt, &els_vectype))
9812 : return false;
9813 : }
9814 :
9815 741283 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9816 741283 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9817 :
9818 741283 : if (loop_vinfo)
9819 : {
9820 511759 : loop = LOOP_VINFO_LOOP (loop_vinfo);
9821 511759 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
9822 511759 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9823 : }
9824 : else
9825 : vf = 1;
9826 :
9827 741283 : vec_num = vect_get_num_copies (vinfo, slp_node);
9828 :
9829 : /* FORNOW. This restriction should be relaxed. */
9830 741283 : if (nested_in_vect_loop && vec_num > 1)
9831 : {
9832 319 : if (dump_enabled_p ())
9833 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9834 : "multiple types in nested loop.\n");
9835 : return false;
9836 : }
9837 :
9838 740964 : elem_type = TREE_TYPE (vectype);
9839 740964 : mode = TYPE_MODE (vectype);
9840 :
9841 : /* FORNOW. In some cases can vectorize even if data-type not supported
9842 : (e.g. - data copies). */
9843 740964 : if (!can_implement_p (mov_optab, mode))
9844 : {
9845 0 : if (dump_enabled_p ())
9846 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9847 : "Aligned load, but unsupported type.\n");
9848 : return false;
9849 : }
9850 :
9851 : /* Check if the load is a part of an interleaving chain. */
9852 740964 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
9853 : {
9854 324432 : grouped_load = true;
9855 : /* FORNOW */
9856 324432 : gcc_assert (!nested_in_vect_loop);
9857 324432 : gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
9858 :
9859 324432 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
9860 324432 : group_size = DR_GROUP_SIZE (first_stmt_info);
9861 :
9862 : /* Invalidate assumptions made by dependence analysis when vectorization
9863 : on the unrolled body effectively re-orders stmts. */
9864 324432 : if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
9865 324432 : && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
9866 : STMT_VINFO_MIN_NEG_DIST (stmt_info)))
9867 : {
9868 12 : if (dump_enabled_p ())
9869 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9870 : "cannot perform implicit CSE when performing "
9871 : "group loads with negative dependence distance\n");
9872 : return false;
9873 : }
9874 : }
9875 : else
9876 : group_size = 1;
9877 :
9878 740952 : vect_load_store_data _ls_data{};
9879 740952 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
9880 740952 : if (cost_vec
9881 740952 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
9882 : VLS_LOAD, &ls))
9883 : return false;
9884 : /* Temporary aliases to analysis data, should not be modified through
9885 : these. */
9886 627552 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
9887 627552 : const dr_alignment_support alignment_support_scheme
9888 : = ls.alignment_support_scheme;
9889 627552 : const int misalignment = ls.misalignment;
9890 627552 : const poly_int64 poffset = ls.poffset;
9891 627552 : const vec<int> &elsvals = ls.elsvals;
9892 :
9893 627552 : int maskload_elsval = 0;
9894 627552 : bool need_zeroing = false;
9895 :
9896 : /* We might need to explicitly zero inactive elements if there are
9897 : padding bits in the type that might leak otherwise.
9898 : Refer to PR115336. */
9899 627552 : tree scalar_type = TREE_TYPE (scalar_dest);
9900 627552 : bool type_mode_padding_p
9901 1255104 : = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
9902 :
9903 627552 : if (slp_node->ldst_lanes
9904 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
9905 : {
9906 0 : if (dump_enabled_p ())
9907 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9908 : "discovered load-lane but cannot use it.\n");
9909 : return false;
9910 : }
9911 :
9912 627552 : if (mask_node)
9913 : {
9914 2960 : if (memory_access_type == VMAT_CONTIGUOUS)
9915 : {
9916 2079 : machine_mode vec_mode = TYPE_MODE (vectype);
9917 720 : if (!VECTOR_MODE_P (vec_mode)
9918 4158 : || !can_vec_mask_load_store_p (vec_mode,
9919 2079 : TYPE_MODE (mask_vectype),
9920 : true, NULL, &ls.elsvals))
9921 : return false;
9922 : }
9923 881 : else if (memory_access_type == VMAT_ELEMENTWISE
9924 881 : || memory_access_type == VMAT_STRIDED_SLP)
9925 : {
9926 0 : if (dump_enabled_p ())
9927 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9928 : "unsupported masked strided access.\n");
9929 : return false;
9930 : }
9931 881 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
9932 881 : && !mat_gather_scatter_p (memory_access_type))
9933 : {
9934 62 : if (dump_enabled_p ())
9935 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9936 : "unsupported access type for masked load.\n");
9937 : return false;
9938 : }
9939 819 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
9940 : {
9941 488 : if (dump_enabled_p ())
9942 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9943 : "unsupported masked emulated gather.\n");
9944 : return false;
9945 : }
9946 : }
9947 :
9948 626651 : bool costing_p = cost_vec;
9949 :
9950 626651 : if (costing_p) /* transformation not required. */
9951 : {
9952 456864 : if (loop_vinfo
9953 324123 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
9954 219630 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
9955 : VLS_LOAD, group_size, &ls,
9956 : mask_node, &ls.elsvals);
9957 :
9958 : /* If the type needs padding we must zero inactive elements.
9959 : Check if we can do that with a VEC_COND_EXPR and store the
9960 : elsval we choose in MASKLOAD_ELSVAL. */
9961 456864 : if (ls.elsvals.length ()
9962 61084 : && type_mode_padding_p
9963 7 : && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
9964 61084 : && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
9965 : {
9966 0 : if (dump_enabled_p ())
9967 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9968 : "cannot zero inactive elements.\n");
9969 : return false;
9970 : }
9971 :
9972 456864 : if (mask_node
9973 456864 : && !vect_maybe_update_slp_op_vectype (mask_node,
9974 : mask_vectype))
9975 : {
9976 0 : if (dump_enabled_p ())
9977 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9978 : "incompatible vector types for invariants\n");
9979 : return false;
9980 : }
9981 :
9982 456864 : if (dump_enabled_p ()
9983 25815 : && memory_access_type != VMAT_ELEMENTWISE
9984 25704 : && !mat_gather_scatter_p (memory_access_type)
9985 25389 : && memory_access_type != VMAT_STRIDED_SLP
9986 25389 : && memory_access_type != VMAT_INVARIANT
9987 481323 : && alignment_support_scheme != dr_aligned)
9988 10034 : dump_printf_loc (MSG_NOTE, vect_location,
9989 : "Vectorizing an unaligned access.\n");
9990 :
9991 456864 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
9992 0 : vinfo->any_known_not_updated_vssa = true;
9993 : }
9994 :
9995 : /* For now just use the first available else value.
9996 : get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
9997 : select it here if it is supported. */
9998 626651 : if (elsvals.length ())
9999 84320 : maskload_elsval = *elsvals.begin ();
10000 :
10001 626651 : if (dump_enabled_p () && !costing_p)
10002 16740 : dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
10003 :
10004 : /* Transform. */
10005 :
10006 626651 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
10007 626651 : if (!costing_p)
10008 169787 : ensure_base_align (dr_info);
10009 :
10010 626651 : if (memory_access_type == VMAT_INVARIANT)
10011 : {
10012 4160 : gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
10013 : /* If we have versioned for aliasing or the loop doesn't
10014 : have any data dependencies that would preclude this,
10015 : then we are sure this is a loop invariant load and
10016 : thus we can insert it on the preheader edge.
10017 : TODO: hoist_defs_of_uses should ideally be computed
10018 : once at analysis time, remembered and used in the
10019 : transform time. */
10020 8320 : bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
10021 4160 : && !nested_in_vect_loop);
10022 :
10023 4160 : bool uniform_p = true;
10024 17418 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10025 : {
10026 : /* It is unsafe to hoist a conditional load over the conditions that
10027 : make it valid. When early break this means that any invariant load
10028 : can't be hoisted unless it's in the loop header or if we know
10029 : something else has verified the load is valid to do. Alignment
10030 : peeling would do this since getting through the prologue means the
10031 : load was done at least once and so the vector main body is free to
10032 : hoist it. However today GCC will hoist the load above the PFA
10033 : loop. As such that makes it still invalid and so we can't allow it
10034 : today. */
10035 4938 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10036 1114 : && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
10037 6016 : && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
10038 1078 : != loop->header)
10039 : {
10040 970 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
10041 970 : && dump_enabled_p ())
10042 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10043 : "not hoisting invariant load due to early break"
10044 : "constraints\n");
10045 964 : else if (dump_enabled_p ())
10046 26 : dump_printf_loc (MSG_NOTE, vect_location,
10047 : "not hoisting invariant load due to early break"
10048 : "constraints\n");
10049 : hoist_p = false;
10050 : }
10051 :
10052 4000 : hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
10053 4938 : if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
10054 279 : uniform_p = false;
10055 : }
10056 4160 : if (costing_p)
10057 : {
10058 3324 : if (!uniform_p && (!hoist_p || !vf.is_constant ()))
10059 : {
10060 0 : if (dump_enabled_p ())
10061 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10062 : "not vectorizing non-uniform invariant "
10063 : "load\n");
10064 : return false;
10065 : }
10066 1480 : enum vect_cost_model_location cost_loc
10067 3324 : = hoist_p ? vect_prologue : vect_body;
10068 3324 : unsigned int cost = record_stmt_cost (cost_vec, 1, scalar_load,
10069 : slp_node, 0, cost_loc);
10070 3324 : cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
10071 : slp_node, 0, cost_loc);
10072 3324 : unsigned int prologue_cost = hoist_p ? cost : 0;
10073 1480 : unsigned int inside_cost = hoist_p ? 0 : cost;
10074 3324 : if (dump_enabled_p ())
10075 543 : dump_printf_loc (MSG_NOTE, vect_location,
10076 : "vect_model_load_cost: inside_cost = %d, "
10077 : "prologue_cost = %d .\n",
10078 : inside_cost, prologue_cost);
10079 3324 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10080 3324 : slp_node->data = new vect_load_store_data (std::move (ls));
10081 3324 : return true;
10082 : }
10083 836 : if (hoist_p)
10084 : {
10085 : /* ??? For non-uniform lanes there could be still duplicates.
10086 : We're leaving those to post-vectorizer CSE for the moment. */
10087 634 : auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
10088 2043 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10089 : {
10090 724 : gassign *stmt = as_a <gassign *> (sinfo->stmt);
10091 724 : if (dump_enabled_p ())
10092 352 : dump_printf_loc (MSG_NOTE, vect_location,
10093 : "hoisting out of the vectorized loop: %G",
10094 : (gimple *) stmt);
10095 724 : scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
10096 724 : tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
10097 724 : edge pe = loop_preheader_edge (loop);
10098 724 : gphi *vphi = get_virtual_phi (loop->header);
10099 724 : tree vuse;
10100 724 : if (vphi)
10101 718 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
10102 : else
10103 6 : vuse = gimple_vuse (gsi_stmt (*gsi));
10104 724 : gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
10105 724 : gimple_set_vuse (new_stmt, vuse);
10106 724 : gsi_insert_on_edge_immediate (pe, new_stmt);
10107 724 : hoist_defs_of_uses (new_stmt, loop, true);
10108 724 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
10109 724 : TREE_TYPE (scalar_dest)))
10110 : {
10111 14 : tree tem = make_ssa_name (TREE_TYPE (vectype));
10112 14 : new_stmt = gimple_build_assign (tem,
10113 : NOP_EXPR, scalar_dest);
10114 14 : gsi_insert_on_edge_immediate (pe, new_stmt);
10115 14 : scalar_dest = tem;
10116 : }
10117 724 : scalar_defs.quick_push (scalar_dest);
10118 724 : if (uniform_p)
10119 : break;
10120 : }
10121 634 : if (!uniform_p)
10122 : {
10123 51 : unsigned const_nunits
10124 51 : = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10125 116 : for (j = 0; j < (int) vec_num; ++j)
10126 : {
10127 65 : vec<constructor_elt, va_gc> *v = NULL;
10128 65 : vec_safe_reserve (v, const_nunits, true);
10129 434 : for (unsigned i = 0; i < const_nunits; ++i)
10130 : {
10131 304 : unsigned def_idx
10132 304 : = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
10133 304 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
10134 : scalar_defs[def_idx]);
10135 : }
10136 65 : scalar_dest = build_constructor (vectype, v);
10137 65 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10138 : vectype, NULL);
10139 65 : slp_node->push_vec_def (new_temp);
10140 : }
10141 51 : return true;
10142 : }
10143 583 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10144 : vectype, NULL);
10145 634 : }
10146 : else
10147 : {
10148 202 : gcc_assert (uniform_p);
10149 202 : gimple_stmt_iterator gsi2 = *gsi;
10150 202 : gsi_next (&gsi2);
10151 202 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10152 : vectype, &gsi2);
10153 : }
10154 1646 : for (j = 0; j < (int) vec_num; ++j)
10155 861 : slp_node->push_vec_def (new_temp);
10156 : return true;
10157 : }
10158 :
10159 622491 : if (memory_access_type == VMAT_ELEMENTWISE
10160 622491 : || memory_access_type == VMAT_STRIDED_SLP)
10161 : {
10162 23762 : gimple_stmt_iterator incr_gsi;
10163 23762 : bool insert_after;
10164 23762 : tree offvar = NULL_TREE;
10165 23762 : tree ivstep;
10166 23762 : tree running_off;
10167 23762 : vec<constructor_elt, va_gc> *v = NULL;
10168 23762 : tree stride_base, stride_step = NULL_TREE, alias_off;
10169 : /* Checked by get_load_store_type. */
10170 23762 : unsigned int const_nunits = nunits.to_constant ();
10171 23762 : unsigned HOST_WIDE_INT cst_offset = 0;
10172 23762 : tree dr_offset;
10173 23762 : unsigned int inside_cost = 0;
10174 :
10175 23762 : gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
10176 23762 : gcc_assert (!nested_in_vect_loop);
10177 :
10178 23762 : if (grouped_load)
10179 : {
10180 : /* If we elided a consecutive load permutation, don't
10181 : use the original first statement (which could be elided)
10182 : but the one the load permutation starts with.
10183 : This ensures the stride_base below is correct. */
10184 10783 : if (!ls.subchain_p)
10185 10739 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10186 : else
10187 44 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10188 10783 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10189 10783 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10190 : }
10191 : else
10192 : {
10193 12979 : first_stmt_info = stmt_info;
10194 12979 : first_dr_info = dr_info;
10195 12979 : ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
10196 : }
10197 :
10198 23762 : if (grouped_load)
10199 : {
10200 10783 : if (memory_access_type == VMAT_STRIDED_SLP)
10201 : {
10202 : /* If we elided a consecutive load permutation, adjust
10203 : the group size here. */
10204 4217 : if (!ls.subchain_p)
10205 4173 : group_size = DR_GROUP_SIZE (first_stmt_info);
10206 : else
10207 44 : group_size = SLP_TREE_LANES (slp_node);
10208 : }
10209 : else /* VMAT_ELEMENTWISE */
10210 6566 : group_size = SLP_TREE_LANES (slp_node);
10211 : }
10212 : else
10213 : group_size = 1;
10214 :
10215 23762 : if (!costing_p)
10216 : {
10217 3450 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
10218 3450 : stride_base = fold_build_pointer_plus (
10219 : DR_BASE_ADDRESS (first_dr_info->dr),
10220 : size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
10221 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
10222 3450 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
10223 :
10224 : /* For a load with loop-invariant (but other than power-of-2)
10225 : stride (i.e. not a grouped access) like so:
10226 :
10227 : for (i = 0; i < n; i += stride)
10228 : ... = array[i];
10229 :
10230 : we generate a new induction variable and new accesses to
10231 : form a new vector (or vectors, depending on ncopies):
10232 :
10233 : for (j = 0; ; j += VF*stride)
10234 : tmp1 = array[j];
10235 : tmp2 = array[j + stride];
10236 : ...
10237 : vectemp = {tmp1, tmp2, ...}
10238 : */
10239 :
10240 3450 : tree increment = fold_convert (TREE_TYPE (stride_step),
10241 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
10242 3450 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
10243 : stride_step, increment);
10244 :
10245 3450 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
10246 :
10247 3450 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
10248 3450 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10249 3450 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
10250 : else
10251 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
10252 : true, NULL_TREE, true,
10253 : GSI_SAME_STMT);
10254 3450 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
10255 : insert_after, &offvar, NULL, true);
10256 :
10257 3450 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
10258 : }
10259 :
10260 23762 : running_off = offvar;
10261 23762 : alias_off = build_int_cst (ref_type, 0);
10262 23762 : int nloads = const_nunits;
10263 23762 : int lnel = 1;
10264 23762 : tree ltype = TREE_TYPE (vectype);
10265 23762 : tree lvectype = vectype;
10266 23762 : auto_vec<tree> dr_chain;
10267 : /* ??? Modify local copies of alignment_support_scheme and
10268 : misalignment, but this part of analysis should be done
10269 : earlier and remembered, likewise the chosen load mode. */
10270 23762 : const dr_alignment_support tem = alignment_support_scheme;
10271 23762 : dr_alignment_support alignment_support_scheme = tem;
10272 23762 : const int tem2 = misalignment;
10273 23762 : int misalignment = tem2;
10274 23762 : if (memory_access_type == VMAT_STRIDED_SLP)
10275 : {
10276 17196 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
10277 : /* Use the target vector type if the group size is a multiple
10278 : of it. */
10279 17196 : if (n == const_nunits)
10280 : {
10281 2247 : int mis_align = dr_misalignment (first_dr_info, vectype);
10282 : /* With VF > 1 we advance the DR by step, if that is constant
10283 : and only aligned when performed VF times, DR alignment
10284 : analysis can analyze this as aligned since it assumes
10285 : contiguous accesses. But that is not how we code generate
10286 : here, so adjust for this. */
10287 2247 : if (maybe_gt (vf, 1u)
10288 3613 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10289 3401 : DR_TARGET_ALIGNMENT (first_dr_info)))
10290 212 : mis_align = -1;
10291 2247 : dr_alignment_support dr_align
10292 2247 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
10293 : mis_align);
10294 2247 : if (dr_align == dr_aligned
10295 2247 : || dr_align == dr_unaligned_supported)
10296 : {
10297 17196 : nloads = 1;
10298 17196 : lnel = const_nunits;
10299 17196 : ltype = vectype;
10300 17196 : alignment_support_scheme = dr_align;
10301 17196 : misalignment = mis_align;
10302 : }
10303 : }
10304 : /* Else use the biggest vector we can load the group without
10305 : accessing excess elements. */
10306 14949 : else if (n > 1)
10307 : {
10308 1965 : tree ptype;
10309 1965 : tree vtype
10310 1965 : = vector_vector_composition_type (vectype, const_nunits / n,
10311 : &ptype);
10312 1965 : if (vtype != NULL_TREE)
10313 : {
10314 1927 : dr_alignment_support dr_align;
10315 1927 : int mis_align = 0;
10316 1927 : if (VECTOR_TYPE_P (ptype))
10317 : {
10318 1005 : mis_align = dr_misalignment (first_dr_info, ptype);
10319 1005 : if (maybe_gt (vf, 1u)
10320 1980 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10321 1011 : DR_TARGET_ALIGNMENT (first_dr_info)))
10322 969 : mis_align = -1;
10323 1005 : dr_align
10324 1005 : = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
10325 : mis_align);
10326 : }
10327 : else
10328 : dr_align = dr_unaligned_supported;
10329 1927 : if (dr_align == dr_aligned
10330 1927 : || dr_align == dr_unaligned_supported)
10331 : {
10332 1927 : nloads = const_nunits / n;
10333 1927 : lnel = n;
10334 1927 : lvectype = vtype;
10335 1927 : ltype = ptype;
10336 1927 : alignment_support_scheme = dr_align;
10337 1927 : misalignment = mis_align;
10338 : }
10339 : }
10340 : }
10341 17196 : unsigned align;
10342 17196 : if (alignment_support_scheme == dr_aligned)
10343 20 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
10344 : else
10345 17176 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
10346 : /* Alignment is at most the access size if we do multiple loads. */
10347 17196 : if (nloads > 1)
10348 14949 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
10349 17196 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
10350 : }
10351 :
10352 23762 : if (costing_p)
10353 : {
10354 : /* Record the composition type for target access during costing. */
10355 20312 : ls.ls_type = lvectype;
10356 20312 : ls.ls_eltype = ltype;
10357 : }
10358 : else
10359 3450 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
10360 :
10361 : /* For SLP permutation support we need to load the whole group,
10362 : not only the number of vector stmts the permutation result
10363 : fits in. */
10364 23762 : int ncopies;
10365 23762 : if (ls.slp_perm)
10366 : {
10367 2869 : gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
10368 : /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10369 : variable VF. */
10370 2869 : unsigned int const_vf = vf.to_constant ();
10371 2869 : ncopies = CEIL (group_size * const_vf, const_nunits);
10372 2869 : dr_chain.create (ncopies);
10373 : }
10374 : else
10375 : ncopies = vec_num;
10376 :
10377 23762 : unsigned int group_el = 0;
10378 23762 : unsigned HOST_WIDE_INT
10379 23762 : elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
10380 23762 : unsigned int n_groups = 0;
10381 : /* For costing some adjacent vector loads, we'd like to cost with
10382 : the total number of them once instead of cost each one by one. */
10383 23762 : unsigned int n_adjacent_loads = 0;
10384 56762 : for (j = 0; j < ncopies; j++)
10385 : {
10386 33000 : if (nloads > 1 && !costing_p)
10387 3165 : vec_alloc (v, nloads);
10388 33000 : gimple *new_stmt = NULL;
10389 139259 : for (i = 0; i < nloads; i++)
10390 : {
10391 106259 : if (costing_p)
10392 : {
10393 : /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10394 : avoid ICE, see PR110776. */
10395 96158 : if (VECTOR_TYPE_P (ltype)
10396 5822 : && memory_access_type != VMAT_ELEMENTWISE)
10397 5822 : n_adjacent_loads++;
10398 : else
10399 90336 : inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
10400 : slp_node, 0, vect_body);
10401 96158 : continue;
10402 : }
10403 10101 : unsigned int load_el = group_el;
10404 : /* For elementwise accesses apply a load permutation directly. */
10405 10101 : if (memory_access_type == VMAT_ELEMENTWISE
10406 10101 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10407 2062 : load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
10408 10101 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
10409 10101 : load_el * elsz + cst_offset);
10410 10101 : tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
10411 10101 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
10412 10101 : new_temp = make_ssa_name (ltype);
10413 10101 : new_stmt = gimple_build_assign (new_temp, data_ref);
10414 10101 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
10415 10101 : if (nloads > 1)
10416 8494 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
10417 :
10418 10101 : group_el += lnel;
10419 10101 : if (group_el == group_size)
10420 : {
10421 9754 : n_groups++;
10422 : /* When doing SLP make sure to not load elements from
10423 : the next vector iteration, those will not be accessed
10424 : so just use the last element again. See PR107451. */
10425 9754 : if (known_lt (n_groups, vf))
10426 : {
10427 6284 : tree newoff = copy_ssa_name (running_off);
10428 6284 : gimple *incr
10429 6284 : = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
10430 : running_off, stride_step);
10431 6284 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
10432 6284 : running_off = newoff;
10433 : }
10434 : group_el = 0;
10435 : }
10436 : }
10437 :
10438 33000 : if (nloads > 1)
10439 : {
10440 24263 : if (costing_p)
10441 21098 : inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
10442 : slp_node, 0, vect_body);
10443 : else
10444 : {
10445 3165 : tree vec_inv = build_constructor (lvectype, v);
10446 3165 : new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
10447 : lvectype, gsi);
10448 3165 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
10449 3165 : if (lvectype != vectype)
10450 : {
10451 398 : new_stmt
10452 398 : = gimple_build_assign (make_ssa_name (vectype),
10453 : VIEW_CONVERT_EXPR,
10454 : build1 (VIEW_CONVERT_EXPR,
10455 : vectype, new_temp));
10456 398 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10457 : gsi);
10458 : }
10459 : }
10460 : }
10461 8737 : else if (!costing_p && ltype != vectype)
10462 : {
10463 1588 : new_stmt = gimple_build_assign (make_ssa_name (vectype),
10464 : VIEW_CONVERT_EXPR,
10465 : build1 (VIEW_CONVERT_EXPR,
10466 : vectype, new_temp));
10467 1588 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10468 : gsi);
10469 : }
10470 :
10471 33000 : if (!costing_p)
10472 : {
10473 4772 : if (ls.slp_perm)
10474 1682 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
10475 : else
10476 3090 : slp_node->push_vec_def (new_stmt);
10477 : }
10478 : }
10479 23762 : if (ls.slp_perm)
10480 : {
10481 2869 : if (costing_p)
10482 : {
10483 2076 : gcc_assert (ls.n_perms != -1U);
10484 2076 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
10485 : slp_node, 0, vect_body);
10486 : }
10487 : else
10488 : {
10489 793 : unsigned n_perms2;
10490 793 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
10491 : false, &n_perms2);
10492 793 : gcc_assert (ls.n_perms == n_perms2);
10493 : }
10494 : }
10495 :
10496 23762 : if (costing_p)
10497 : {
10498 20312 : if (n_adjacent_loads > 0)
10499 2152 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10500 : alignment_support_scheme, misalignment, false,
10501 : &inside_cost, nullptr, cost_vec, cost_vec,
10502 : true);
10503 20312 : if (dump_enabled_p ())
10504 498 : dump_printf_loc (MSG_NOTE, vect_location,
10505 : "vect_model_load_cost: inside_cost = %u, "
10506 : "prologue_cost = 0 .\n",
10507 : inside_cost);
10508 20312 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10509 20312 : slp_node->data = new vect_load_store_data (std::move (ls));
10510 : }
10511 :
10512 23762 : return true;
10513 23762 : }
10514 :
10515 598729 : if (mat_gather_scatter_p (memory_access_type)
10516 598729 : && !ls.ls_type)
10517 : grouped_load = false;
10518 :
10519 595738 : if (grouped_load
10520 598729 : || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10521 : {
10522 276125 : if (grouped_load)
10523 : {
10524 275677 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10525 275677 : group_size = DR_GROUP_SIZE (first_stmt_info);
10526 : }
10527 : else
10528 : {
10529 : first_stmt_info = stmt_info;
10530 : group_size = 1;
10531 : }
10532 : /* For SLP vectorization we directly vectorize a subchain
10533 : without permutation. */
10534 276125 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10535 215650 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10536 : /* For BB vectorization always use the first stmt to base
10537 : the data ref pointer on. */
10538 276125 : if (bb_vinfo)
10539 221383 : first_stmt_info_for_drptr
10540 221383 : = vect_find_first_scalar_stmt_in_slp (slp_node);
10541 :
10542 276125 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10543 276125 : group_gap_adj = 0;
10544 :
10545 : /* VEC_NUM is the number of vect stmts to be created for this group. */
10546 276125 : grouped_load = false;
10547 : /* If an SLP permutation is from N elements to N elements,
10548 : and if one vector holds a whole number of N, we can load
10549 : the inputs to the permutation in the same way as an
10550 : unpermuted sequence. In other cases we need to load the
10551 : whole group, not only the number of vector stmts the
10552 : permutation result fits in. */
10553 276125 : unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
10554 276125 : if (nested_in_vect_loop)
10555 : /* We do not support grouped accesses in a nested loop,
10556 : instead the access is contiguous but it might be
10557 : permuted. No gap adjustment is needed though. */
10558 : ;
10559 276123 : else if (ls.slp_perm
10560 276123 : && (group_size != scalar_lanes
10561 11608 : || !multiple_p (nunits, group_size)))
10562 : {
10563 : /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10564 : variable VF; see vect_transform_slp_perm_load. */
10565 50055 : unsigned int const_vf = vf.to_constant ();
10566 50055 : unsigned int const_nunits = nunits.to_constant ();
10567 50055 : vec_num = CEIL (group_size * const_vf, const_nunits);
10568 50055 : group_gap_adj = vf * group_size - nunits * vec_num;
10569 : }
10570 : else
10571 : {
10572 226068 : group_gap_adj = group_size - scalar_lanes;
10573 : }
10574 :
10575 276125 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10576 : }
10577 : else
10578 : {
10579 322604 : first_stmt_info = stmt_info;
10580 322604 : first_dr_info = dr_info;
10581 322604 : group_size = 1;
10582 322604 : group_gap_adj = 0;
10583 322604 : ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
10584 : }
10585 :
10586 598729 : vec_loop_masks *loop_masks
10587 377346 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10588 598729 : ? &LOOP_VINFO_MASKS (loop_vinfo)
10589 34 : : NULL);
10590 34 : vec_loop_lens *loop_lens
10591 377346 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
10592 : ? &LOOP_VINFO_LENS (loop_vinfo)
10593 0 : : NULL);
10594 :
10595 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10596 : are some difference here. We cannot enable both the lens and masks
10597 : during transform but it is allowed during analysis.
10598 : Shouldn't go with length-based approach if fully masked. */
10599 598729 : if (cost_vec == NULL)
10600 : /* The cost_vec is NULL during transform. */
10601 165501 : gcc_assert ((!loop_lens || !loop_masks));
10602 :
10603 : /* Targets with store-lane instructions must not require explicit
10604 : realignment. vect_supportable_dr_alignment always returns either
10605 : dr_aligned or dr_unaligned_supported for (non-length) masked
10606 : operations. */
10607 598729 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
10608 : && !mask_node
10609 : && !loop_masks)
10610 : || mat_gather_scatter_p (memory_access_type)
10611 : || alignment_support_scheme == dr_aligned
10612 : || alignment_support_scheme == dr_unaligned_supported);
10613 :
10614 : /* In case the vectorization factor (VF) is bigger than the number
10615 : of elements that we can fit in a vectype (nunits), we have to generate
10616 : more than one vector stmt - i.e - we need to "unroll" the
10617 : vector stmt by a factor VF/nunits. In doing so, we record a pointer
10618 : from one copy of the vector stmt to the next, in the field
10619 : STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10620 : stages to find the correct vector defs to be used when vectorizing
10621 : stmts that use the defs of the current stmt. The example below
10622 : illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10623 : need to create 4 vectorized stmts):
10624 :
10625 : before vectorization:
10626 : RELATED_STMT VEC_STMT
10627 : S1: x = memref - -
10628 : S2: z = x + 1 - -
10629 :
10630 : step 1: vectorize stmt S1:
10631 : We first create the vector stmt VS1_0, and, as usual, record a
10632 : pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10633 : Next, we create the vector stmt VS1_1, and record a pointer to
10634 : it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10635 : Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10636 : stmts and pointers:
10637 : RELATED_STMT VEC_STMT
10638 : VS1_0: vx0 = memref0 VS1_1 -
10639 : VS1_1: vx1 = memref1 VS1_2 -
10640 : VS1_2: vx2 = memref2 VS1_3 -
10641 : VS1_3: vx3 = memref3 - -
10642 : S1: x = load - VS1_0
10643 : S2: z = x + 1 - -
10644 : */
10645 :
10646 : /* If the data reference is aligned (dr_aligned) or potentially unaligned
10647 : on a target that supports unaligned accesses (dr_unaligned_supported)
10648 : we generate the following code:
10649 : p = initial_addr;
10650 : indx = 0;
10651 : loop {
10652 : p = p + indx * vectype_size;
10653 : vec_dest = *(p);
10654 : indx = indx + 1;
10655 : }
10656 :
10657 : Otherwise, the data reference is potentially unaligned on a target that
10658 : does not support unaligned accesses (dr_explicit_realign_optimized) -
10659 : then generate the following code, in which the data in each iteration is
10660 : obtained by two vector loads, one from the previous iteration, and one
10661 : from the current iteration:
10662 : p1 = initial_addr;
10663 : msq_init = *(floor(p1))
10664 : p2 = initial_addr + VS - 1;
10665 : realignment_token = call target_builtin;
10666 : indx = 0;
10667 : loop {
10668 : p2 = p2 + indx * vectype_size
10669 : lsq = *(floor(p2))
10670 : vec_dest = realign_load (msq, lsq, realignment_token)
10671 : indx = indx + 1;
10672 : msq = lsq;
10673 : } */
10674 :
10675 : /* If the misalignment remains the same throughout the execution of the
10676 : loop, we can create the init_addr and permutation mask at the loop
10677 : preheader. Otherwise, it needs to be created inside the loop.
10678 : This can only occur when vectorizing memory accesses in the inner-loop
10679 : nested within an outer-loop that is being vectorized. */
10680 :
10681 598729 : if (nested_in_vect_loop
10682 598729 : && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
10683 1234 : GET_MODE_SIZE (TYPE_MODE (vectype))))
10684 : {
10685 195 : gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
10686 : compute_in_loop = true;
10687 : }
10688 :
10689 598729 : bool diff_first_stmt_info
10690 598729 : = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
10691 :
10692 598729 : tree offset = NULL_TREE;
10693 598729 : if ((alignment_support_scheme == dr_explicit_realign_optimized
10694 598729 : || alignment_support_scheme == dr_explicit_realign)
10695 0 : && !compute_in_loop)
10696 : {
10697 : /* If we have different first_stmt_info, we can't set up realignment
10698 : here, since we can't guarantee first_stmt_info DR has been
10699 : initialized yet, use first_stmt_info_for_drptr DR by bumping the
10700 : distance from first_stmt_info DR instead as below. */
10701 0 : if (!costing_p)
10702 : {
10703 0 : if (!diff_first_stmt_info)
10704 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
10705 : &realignment_token,
10706 : alignment_support_scheme, NULL_TREE,
10707 : &at_loop);
10708 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
10709 : {
10710 0 : phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
10711 0 : offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
10712 : size_one_node);
10713 0 : gcc_assert (!first_stmt_info_for_drptr);
10714 : }
10715 : }
10716 : }
10717 : else
10718 598729 : at_loop = loop;
10719 :
10720 598729 : if (!known_eq (poffset, 0))
10721 4630 : offset = (offset
10722 4630 : ? size_binop (PLUS_EXPR, offset, size_int (poffset))
10723 4630 : : size_int (poffset));
10724 :
10725 598729 : tree dr_increment;
10726 598729 : tree dr_bump;
10727 598729 : tree vec_offset = NULL_TREE;
10728 :
10729 598729 : auto_vec<tree> vec_offsets;
10730 598729 : auto_vec<tree> vec_masks;
10731 598729 : if (mask_node && !costing_p)
10732 628 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
10733 : &vec_masks);
10734 :
10735 598729 : tree vec_mask = NULL_TREE;
10736 598729 : tree vec_els = NULL_TREE;
10737 598729 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10738 : {
10739 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
10740 :
10741 0 : gcc_assert (alignment_support_scheme == dr_aligned
10742 : || alignment_support_scheme == dr_unaligned_supported);
10743 :
10744 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
10745 0 : if (!costing_p)
10746 : {
10747 0 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
10748 : memory_access_type);
10749 0 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
10750 : memory_access_type);
10751 : }
10752 :
10753 0 : unsigned int inside_cost = 0, prologue_cost = 0;
10754 : /* For costing some adjacent vector loads, we'd like to cost with
10755 : the total number of them once instead of cost each one by one. */
10756 0 : unsigned int n_adjacent_loads = 0;
10757 0 : int ncopies = vec_num / group_size;
10758 0 : for (j = 0; j < ncopies; j++)
10759 : {
10760 0 : if (costing_p)
10761 : {
10762 : /* An IFN_LOAD_LANES will load all its vector results,
10763 : regardless of which ones we actually need. Account
10764 : for the cost of unused results. */
10765 0 : if (first_stmt_info == stmt_info)
10766 : {
10767 0 : unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
10768 0 : stmt_vec_info next_stmt_info = first_stmt_info;
10769 0 : do
10770 : {
10771 0 : gaps -= 1;
10772 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
10773 : }
10774 0 : while (next_stmt_info);
10775 0 : if (gaps)
10776 : {
10777 0 : if (dump_enabled_p ())
10778 0 : dump_printf_loc (MSG_NOTE, vect_location,
10779 : "vect_model_load_cost: %d "
10780 : "unused vectors.\n",
10781 : gaps);
10782 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
10783 : alignment_support_scheme,
10784 : misalignment, false, &inside_cost,
10785 : &prologue_cost, cost_vec, cost_vec,
10786 : true);
10787 : }
10788 : }
10789 0 : n_adjacent_loads++;
10790 0 : continue;
10791 0 : }
10792 :
10793 : /* 1. Create the vector or array pointer update chain. */
10794 0 : if (j == 0)
10795 0 : dataref_ptr
10796 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10797 : at_loop, offset, &dummy, gsi,
10798 : NULL, false, dr_increment);
10799 : else
10800 : {
10801 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
10802 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10803 : stmt_info, dr_bump);
10804 : }
10805 0 : if (mask_node)
10806 0 : vec_mask = vec_masks[j];
10807 :
10808 0 : tree vec_array = create_vector_array (vectype, group_size);
10809 :
10810 0 : tree final_mask = NULL_TREE;
10811 0 : tree final_len = NULL_TREE;
10812 0 : tree bias = NULL_TREE;
10813 0 : if (loop_masks)
10814 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10815 : ncopies, vectype, j);
10816 0 : if (vec_mask)
10817 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
10818 : vec_mask, gsi);
10819 :
10820 0 : if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
10821 : {
10822 0 : if (loop_lens)
10823 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
10824 : ncopies, vectype, j, 1, true);
10825 : else
10826 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
10827 0 : signed char biasval
10828 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
10829 0 : bias = build_int_cst (intQI_type_node, biasval);
10830 0 : if (!final_mask)
10831 : {
10832 0 : mask_vectype = truth_type_for (vectype);
10833 0 : final_mask = build_minus_one_cst (mask_vectype);
10834 : }
10835 : }
10836 :
10837 0 : if (final_mask)
10838 : {
10839 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
10840 0 : if (type_mode_padding_p
10841 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
10842 0 : need_zeroing = true;
10843 : }
10844 :
10845 0 : gcall *call;
10846 0 : if (final_len && final_mask)
10847 : {
10848 : /* Emit:
10849 : VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10850 : VEC_MASK, LEN, BIAS). */
10851 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10852 0 : tree alias_ptr = build_int_cst (ref_type, align);
10853 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
10854 : dataref_ptr, alias_ptr,
10855 : final_mask, vec_els,
10856 : final_len, bias);
10857 : }
10858 0 : else if (final_mask)
10859 : {
10860 : /* Emit:
10861 : VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10862 : VEC_MASK). */
10863 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10864 0 : tree alias_ptr = build_int_cst (ref_type, align);
10865 0 : call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
10866 : dataref_ptr, alias_ptr,
10867 : final_mask, vec_els);
10868 : }
10869 : else
10870 : {
10871 : /* Emit:
10872 : VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10873 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
10874 0 : call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
10875 : }
10876 0 : gimple_call_set_lhs (call, vec_array);
10877 0 : gimple_call_set_nothrow (call, true);
10878 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
10879 :
10880 : /* Extract each vector into an SSA_NAME. */
10881 0 : for (unsigned i = 0; i < group_size; i++)
10882 : {
10883 0 : new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
10884 : vec_array, i, need_zeroing,
10885 : final_mask);
10886 0 : slp_node->push_vec_def (new_temp);
10887 : }
10888 :
10889 : /* Record that VEC_ARRAY is now dead. */
10890 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
10891 : }
10892 :
10893 0 : if (costing_p)
10894 : {
10895 0 : if (n_adjacent_loads > 0)
10896 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10897 : alignment_support_scheme, misalignment, false,
10898 : &inside_cost, &prologue_cost, cost_vec,
10899 : cost_vec, true);
10900 0 : if (dump_enabled_p ())
10901 0 : dump_printf_loc (MSG_NOTE, vect_location,
10902 : "vect_model_load_cost: inside_cost = %u, "
10903 : "prologue_cost = %u .\n",
10904 : inside_cost, prologue_cost);
10905 0 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10906 0 : slp_node->data = new vect_load_store_data (std::move (ls));
10907 : }
10908 :
10909 0 : return true;
10910 : }
10911 :
10912 598729 : if (mat_gather_scatter_p (memory_access_type))
10913 : {
10914 2991 : gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
10915 :
10916 2991 : auto_vec<tree> dr_chain (vec_num);
10917 :
10918 : /* If we pun the original vectype the loads as well as costing, length,
10919 : etc. is performed with the new type. After loading we VIEW_CONVERT
10920 : the data to the original vectype. */
10921 2991 : tree original_vectype = vectype;
10922 2991 : if (ls.ls_type)
10923 0 : vectype = ls.ls_type;
10924 :
10925 : /* 1. Create the vector or array pointer update chain. */
10926 2991 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10927 : {
10928 2991 : aggr_type = NULL_TREE;
10929 2991 : dr_increment = NULL_TREE;
10930 2991 : if (!costing_p)
10931 764 : vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
10932 : &vec_offsets);
10933 : }
10934 : else
10935 : {
10936 0 : aggr_type = elem_type;
10937 0 : if (!costing_p)
10938 : {
10939 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
10940 : ls.strided_offset_vectype,
10941 : loop_vinfo, gsi,
10942 : &dr_increment, &dr_bump,
10943 : &vec_offset);
10944 0 : dataref_ptr
10945 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10946 : at_loop, offset, &dummy, gsi,
10947 : NULL, false, dr_increment);
10948 : }
10949 : }
10950 :
10951 2991 : unsigned int inside_cost = 0, prologue_cost = 0;
10952 :
10953 2991 : gimple *new_stmt = NULL;
10954 6755 : for (i = 0; i < vec_num; i++)
10955 : {
10956 3764 : tree final_mask = NULL_TREE;
10957 3764 : tree final_len = NULL_TREE;
10958 3764 : tree bias = NULL_TREE;
10959 3764 : if (!costing_p)
10960 : {
10961 981 : if (mask_node)
10962 156 : vec_mask = vec_masks[i];
10963 981 : if (loop_masks)
10964 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10965 : vec_num, vectype, i);
10966 981 : if (vec_mask)
10967 156 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
10968 : final_mask, vec_mask, gsi);
10969 :
10970 981 : if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10971 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10972 : stmt_info, dr_bump);
10973 : }
10974 :
10975 : /* 2. Create the vector-load in the loop. */
10976 3764 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
10977 3764 : tree alias_align_ptr = build_int_cst (ref_type, align);
10978 3764 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
10979 : {
10980 0 : if (costing_p)
10981 : {
10982 0 : if (ls.supported_offset_vectype
10983 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
10984 : vec_offset))
10985 0 : inside_cost
10986 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
10987 : slp_node, 0, vect_body);
10988 0 : if (ls.supported_scale)
10989 0 : inside_cost
10990 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
10991 : slp_node, 0, vect_body);
10992 :
10993 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
10994 0 : inside_cost
10995 0 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
10996 : slp_node, 0, vect_body);
10997 0 : continue;
10998 0 : }
10999 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11000 0 : vec_offset = vec_offsets[i];
11001 0 : tree zero = build_zero_cst (vectype);
11002 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11003 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
11004 :
11005 : /* Perform the offset conversion and scaling if necessary. */
11006 0 : if (!strided
11007 0 : && (ls.supported_offset_vectype || ls.supported_scale))
11008 : {
11009 0 : gimple_seq stmts = NULL;
11010 0 : if (ls.supported_offset_vectype)
11011 0 : vec_offset = gimple_convert
11012 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
11013 0 : if (ls.supported_scale)
11014 : {
11015 : /* Only scale the vec_offset if we haven't already. */
11016 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
11017 0 : || i == 0)
11018 : {
11019 0 : tree mult_cst = build_int_cst
11020 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
11021 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
11022 0 : tree mult = build_vector_from_val
11023 0 : (TREE_TYPE (vec_offset), mult_cst);
11024 0 : vec_offset = gimple_build
11025 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
11026 : vec_offset, mult);
11027 : }
11028 0 : scale = size_int (ls.supported_scale);
11029 : }
11030 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11031 : }
11032 :
11033 0 : if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
11034 : {
11035 0 : if (loop_lens)
11036 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11037 : vec_num, vectype, i, 1, true);
11038 : else
11039 0 : final_len = build_int_cst (sizetype,
11040 0 : TYPE_VECTOR_SUBPARTS (vectype));
11041 0 : signed char biasval
11042 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11043 0 : bias = build_int_cst (intQI_type_node, biasval);
11044 0 : if (!final_mask)
11045 : {
11046 0 : mask_vectype = truth_type_for (vectype);
11047 0 : final_mask = build_minus_one_cst (mask_vectype);
11048 : }
11049 : }
11050 :
11051 0 : if (final_mask)
11052 : {
11053 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11054 0 : if (type_mode_padding_p
11055 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11056 0 : need_zeroing = true;
11057 : }
11058 :
11059 0 : gcall *call;
11060 0 : if (final_len && final_mask)
11061 : {
11062 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
11063 0 : call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
11064 : 9, dataref_ptr,
11065 : alias_align_ptr,
11066 : vec_offset, scale, zero,
11067 : final_mask, vec_els,
11068 : final_len, bias);
11069 : else
11070 : /* Non-vector offset indicates that prefer to take
11071 : MASK_LEN_STRIDED_LOAD instead of the
11072 : MASK_LEN_GATHER_LOAD with direct stride arg. */
11073 0 : call = gimple_build_call_internal
11074 0 : (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
11075 : vec_offset, zero, final_mask, vec_els, final_len,
11076 : bias);
11077 : }
11078 0 : else if (final_mask)
11079 0 : call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
11080 : 7, dataref_ptr,
11081 : alias_align_ptr,
11082 : vec_offset, scale,
11083 : zero, final_mask, vec_els);
11084 : else
11085 0 : call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
11086 : dataref_ptr,
11087 : alias_align_ptr,
11088 : vec_offset, scale, zero);
11089 0 : gimple_call_set_nothrow (call, true);
11090 0 : new_stmt = call;
11091 0 : data_ref = NULL_TREE;
11092 : }
11093 3764 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
11094 : {
11095 : /* The builtin decls path for gather is legacy, x86 only. */
11096 858 : gcc_assert (!final_len && nunits.is_constant ());
11097 858 : if (costing_p)
11098 : {
11099 572 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11100 572 : inside_cost
11101 572 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11102 : slp_node, 0, vect_body);
11103 572 : continue;
11104 572 : }
11105 286 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11106 286 : poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
11107 286 : if (known_eq (nunits, offset_nunits))
11108 : {
11109 137 : new_stmt = vect_build_one_gather_load_call
11110 137 : (vinfo, stmt_info, slp_node, vectype, gsi,
11111 137 : ls.gs.decl, dataref_ptr, vec_offsets[i],
11112 : final_mask);
11113 137 : data_ref = NULL_TREE;
11114 : }
11115 149 : else if (known_eq (nunits, offset_nunits * 2))
11116 : {
11117 : /* We have a offset vector with half the number of
11118 : lanes but the builtins will produce full vectype
11119 : data with just the lower lanes filled. */
11120 63 : new_stmt = vect_build_one_gather_load_call
11121 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11122 63 : ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
11123 : final_mask);
11124 63 : tree low = make_ssa_name (vectype);
11125 63 : gimple_set_lhs (new_stmt, low);
11126 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11127 :
11128 : /* now put upper half of final_mask in final_mask low. */
11129 63 : if (final_mask
11130 63 : && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
11131 : {
11132 11 : int count = nunits.to_constant ();
11133 11 : vec_perm_builder sel (count, count, 1);
11134 11 : sel.quick_grow (count);
11135 98 : for (int i = 0; i < count; ++i)
11136 76 : sel[i] = i | (count / 2);
11137 11 : vec_perm_indices indices (sel, 2, count);
11138 11 : tree perm_mask = vect_gen_perm_mask_checked
11139 11 : (TREE_TYPE (final_mask), indices);
11140 11 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11141 : final_mask, final_mask,
11142 : perm_mask);
11143 11 : final_mask = make_ssa_name (TREE_TYPE (final_mask));
11144 11 : gimple_set_lhs (new_stmt, final_mask);
11145 11 : vect_finish_stmt_generation (vinfo, stmt_info,
11146 : new_stmt, gsi);
11147 11 : }
11148 52 : else if (final_mask)
11149 : {
11150 24 : new_stmt = gimple_build_assign (NULL_TREE,
11151 : VEC_UNPACK_HI_EXPR,
11152 : final_mask);
11153 24 : final_mask = make_ssa_name
11154 24 : (truth_type_for (offset_vectype));
11155 24 : gimple_set_lhs (new_stmt, final_mask);
11156 24 : vect_finish_stmt_generation (vinfo, stmt_info,
11157 : new_stmt, gsi);
11158 : }
11159 :
11160 63 : new_stmt = vect_build_one_gather_load_call
11161 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11162 : ls.gs.decl, dataref_ptr,
11163 63 : vec_offsets[2 * i + 1], final_mask);
11164 63 : tree high = make_ssa_name (vectype);
11165 63 : gimple_set_lhs (new_stmt, high);
11166 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11167 :
11168 : /* compose low + high. */
11169 63 : int count = nunits.to_constant ();
11170 63 : vec_perm_builder sel (count, count, 1);
11171 63 : sel.quick_grow (count);
11172 710 : for (int i = 0; i < count; ++i)
11173 584 : sel[i] = i < count / 2 ? i : i + count / 2;
11174 63 : vec_perm_indices indices (sel, 2, count);
11175 63 : tree perm_mask
11176 63 : = vect_gen_perm_mask_checked (vectype, indices);
11177 63 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11178 : low, high, perm_mask);
11179 63 : data_ref = NULL_TREE;
11180 63 : }
11181 86 : else if (known_eq (nunits * 2, offset_nunits))
11182 : {
11183 : /* We have a offset vector with double the number of
11184 : lanes. Select the low/high part accordingly. */
11185 86 : vec_offset = vec_offsets[i / 2];
11186 86 : if (i & 1)
11187 : {
11188 43 : int count = offset_nunits.to_constant ();
11189 43 : vec_perm_builder sel (count, count, 1);
11190 43 : sel.quick_grow (count);
11191 506 : for (int i = 0; i < count; ++i)
11192 420 : sel[i] = i | (count / 2);
11193 43 : vec_perm_indices indices (sel, 2, count);
11194 43 : tree perm_mask = vect_gen_perm_mask_checked
11195 43 : (TREE_TYPE (vec_offset), indices);
11196 43 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11197 : vec_offset, vec_offset,
11198 : perm_mask);
11199 43 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
11200 43 : gimple_set_lhs (new_stmt, vec_offset);
11201 43 : vect_finish_stmt_generation (vinfo, stmt_info,
11202 : new_stmt, gsi);
11203 43 : }
11204 86 : new_stmt = vect_build_one_gather_load_call
11205 86 : (vinfo, stmt_info, slp_node, vectype, gsi,
11206 : ls.gs.decl,
11207 : dataref_ptr, vec_offset, final_mask);
11208 86 : data_ref = NULL_TREE;
11209 : }
11210 : else
11211 0 : gcc_unreachable ();
11212 : }
11213 : else
11214 : {
11215 : /* Emulated gather-scatter. */
11216 2906 : gcc_assert (!final_mask);
11217 2906 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
11218 2906 : if (costing_p)
11219 : {
11220 : /* For emulated gathers N offset vector element
11221 : offset add is consumed by the load). */
11222 2211 : inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
11223 : slp_node, 0, vect_body);
11224 : /* N scalar loads plus gathering them into a
11225 : vector. */
11226 2211 : inside_cost
11227 2211 : = record_stmt_cost (cost_vec, const_nunits, scalar_load,
11228 : slp_node, 0, vect_body);
11229 2211 : inside_cost
11230 2211 : = record_stmt_cost (cost_vec, 1, vec_construct,
11231 : slp_node, 0, vect_body);
11232 2211 : continue;
11233 : }
11234 695 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11235 695 : unsigned HOST_WIDE_INT const_offset_nunits
11236 695 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
11237 695 : vec<constructor_elt, va_gc> *ctor_elts;
11238 695 : vec_alloc (ctor_elts, const_nunits);
11239 695 : gimple_seq stmts = NULL;
11240 : /* We support offset vectors with more elements
11241 : than the data vector for now. */
11242 695 : unsigned HOST_WIDE_INT factor
11243 : = const_offset_nunits / const_nunits;
11244 695 : vec_offset = vec_offsets[i / factor];
11245 695 : unsigned elt_offset = (i % factor) * const_nunits;
11246 695 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
11247 695 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11248 695 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
11249 3512 : for (unsigned k = 0; k < const_nunits; ++k)
11250 : {
11251 2122 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
11252 : bitsize_int (k + elt_offset));
11253 6366 : tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
11254 2122 : vec_offset, TYPE_SIZE (idx_type),
11255 : boff);
11256 2122 : idx = gimple_convert (&stmts, sizetype, idx);
11257 2122 : idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
11258 2122 : tree ptr = gimple_build (&stmts, PLUS_EXPR,
11259 2122 : TREE_TYPE (dataref_ptr),
11260 : dataref_ptr, idx);
11261 2122 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
11262 2122 : tree elt = make_ssa_name (TREE_TYPE (vectype));
11263 2122 : tree ref = build2 (MEM_REF, ltype, ptr,
11264 : build_int_cst (ref_type, 0));
11265 2122 : new_stmt = gimple_build_assign (elt, ref);
11266 4244 : gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
11267 2122 : gimple_seq_add_stmt (&stmts, new_stmt);
11268 2122 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
11269 : }
11270 695 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11271 695 : new_stmt = gimple_build_assign (NULL_TREE,
11272 : build_constructor (vectype,
11273 : ctor_elts));
11274 695 : data_ref = NULL_TREE;
11275 : }
11276 :
11277 981 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11278 : /* DATA_REF is null if we've already built the statement. */
11279 981 : if (data_ref)
11280 : {
11281 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11282 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11283 : }
11284 1962 : new_temp = (need_zeroing
11285 981 : ? make_ssa_name (vectype)
11286 981 : : make_ssa_name (vec_dest, new_stmt));
11287 981 : gimple_set_lhs (new_stmt, new_temp);
11288 981 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11289 :
11290 : /* If we need to explicitly zero inactive elements emit a
11291 : VEC_COND_EXPR that does so. */
11292 981 : if (need_zeroing)
11293 : {
11294 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11295 : vectype);
11296 :
11297 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11298 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11299 : final_mask, new_temp, vec_els);
11300 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11301 0 : new_temp = new_temp2;
11302 : }
11303 :
11304 981 : if (ls.ls_type)
11305 : {
11306 0 : new_stmt = gimple_build_assign (make_ssa_name
11307 : (original_vectype),
11308 : VIEW_CONVERT_EXPR,
11309 : build1 (VIEW_CONVERT_EXPR,
11310 : original_vectype,
11311 : new_temp));
11312 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11313 : }
11314 :
11315 : /* Store vector loads in the corresponding SLP_NODE. */
11316 981 : if (!costing_p)
11317 : {
11318 981 : if (ls.slp_perm)
11319 0 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
11320 : else
11321 981 : slp_node->push_vec_def (new_stmt);
11322 : }
11323 : }
11324 :
11325 2991 : if (ls.slp_perm)
11326 : {
11327 0 : if (costing_p)
11328 : {
11329 0 : gcc_assert (ls.n_perms != -1U);
11330 0 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11331 : slp_node, 0, vect_body);
11332 : }
11333 : else
11334 : {
11335 0 : unsigned n_perms2;
11336 0 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
11337 : false, &n_perms2);
11338 0 : gcc_assert (ls.n_perms == n_perms2);
11339 : }
11340 : }
11341 :
11342 2991 : if (costing_p)
11343 : {
11344 2227 : if (dump_enabled_p ())
11345 315 : dump_printf_loc (MSG_NOTE, vect_location,
11346 : "vect_model_load_cost: inside_cost = %u, "
11347 : "prologue_cost = %u .\n",
11348 : inside_cost, prologue_cost);
11349 2227 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11350 2227 : slp_node->data = new vect_load_store_data (std::move (ls));
11351 : }
11352 2991 : return true;
11353 2991 : }
11354 :
11355 595738 : aggr_type = vectype;
11356 595738 : if (!costing_p)
11357 : {
11358 164737 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
11359 : memory_access_type);
11360 164737 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
11361 : memory_access_type);
11362 : }
11363 :
11364 595738 : poly_uint64 group_elt = 0;
11365 595738 : unsigned int inside_cost = 0, prologue_cost = 0;
11366 : /* For costing some adjacent vector loads, we'd like to cost with
11367 : the total number of them once instead of cost each one by one. */
11368 595738 : unsigned int n_adjacent_loads = 0;
11369 :
11370 : /* 1. Create the vector or array pointer update chain. */
11371 595738 : if (!costing_p)
11372 : {
11373 164737 : bool simd_lane_access_p
11374 164737 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
11375 164737 : if (simd_lane_access_p
11376 1628 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
11377 1628 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
11378 1628 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
11379 1628 : && integer_zerop (DR_INIT (first_dr_info->dr))
11380 1628 : && alias_sets_conflict_p (get_alias_set (aggr_type),
11381 1628 : get_alias_set (TREE_TYPE (ref_type)))
11382 164737 : && (alignment_support_scheme == dr_aligned
11383 1628 : || alignment_support_scheme == dr_unaligned_supported))
11384 : {
11385 1628 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
11386 1628 : dataref_offset = build_int_cst (ref_type, 0);
11387 : }
11388 163109 : else if (diff_first_stmt_info)
11389 : {
11390 4017 : dataref_ptr
11391 4017 : = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
11392 : aggr_type, at_loop, offset, &dummy,
11393 : gsi, NULL, simd_lane_access_p,
11394 : dr_increment);
11395 : /* Adjust the pointer by the difference to first_stmt. */
11396 4017 : data_reference_p ptrdr
11397 : = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
11398 4017 : tree diff = fold_convert (sizetype,
11399 : size_binop (MINUS_EXPR,
11400 : DR_INIT (first_dr_info->dr),
11401 : DR_INIT (ptrdr)));
11402 4017 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11403 : stmt_info, diff);
11404 4017 : if (alignment_support_scheme == dr_explicit_realign)
11405 : {
11406 0 : msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
11407 : vectype, gsi,
11408 : &realignment_token,
11409 : alignment_support_scheme,
11410 : dataref_ptr, &at_loop);
11411 0 : gcc_assert (!compute_in_loop);
11412 : }
11413 : }
11414 : else
11415 159092 : dataref_ptr
11416 159092 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11417 : at_loop,
11418 : offset, &dummy, gsi, NULL,
11419 : simd_lane_access_p, dr_increment);
11420 : }
11421 :
11422 595738 : auto_vec<tree> dr_chain;
11423 595738 : if (grouped_load || ls.slp_perm)
11424 60475 : dr_chain.create (vec_num);
11425 :
11426 595738 : gimple *new_stmt = NULL;
11427 1554596 : for (i = 0; i < vec_num; i++)
11428 : {
11429 958858 : tree final_mask = NULL_TREE;
11430 958858 : tree final_len = NULL_TREE;
11431 958858 : tree bias = NULL_TREE;
11432 :
11433 958858 : if (!costing_p)
11434 : {
11435 256370 : if (mask_node)
11436 651 : vec_mask = vec_masks[i];
11437 256370 : if (loop_masks)
11438 51 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11439 : vec_num, vectype, i);
11440 256370 : if (vec_mask)
11441 651 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11442 : final_mask, vec_mask, gsi);
11443 :
11444 256370 : if (i > 0)
11445 91633 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11446 : dr_bump);
11447 : }
11448 :
11449 : /* 2. Create the vector-load in the loop. */
11450 958858 : switch (alignment_support_scheme)
11451 : {
11452 958858 : case dr_aligned:
11453 958858 : case dr_unaligned_supported:
11454 958858 : {
11455 958858 : if (costing_p)
11456 : break;
11457 :
11458 256370 : unsigned int misalign;
11459 256370 : unsigned HOST_WIDE_INT align;
11460 256370 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
11461 256370 : if (alignment_support_scheme == dr_aligned)
11462 : misalign = 0;
11463 165665 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
11464 : {
11465 125907 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
11466 125907 : misalign = 0;
11467 : }
11468 : else
11469 39758 : misalign = misalignment;
11470 256370 : if (dataref_offset == NULL_TREE
11471 254244 : && TREE_CODE (dataref_ptr) == SSA_NAME)
11472 174412 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
11473 : misalign);
11474 256370 : align = least_bit_hwi (misalign | align);
11475 :
11476 : /* Compute IFN when LOOP_LENS or final_mask valid. */
11477 256370 : machine_mode vmode = TYPE_MODE (vectype);
11478 256370 : machine_mode new_vmode = vmode;
11479 256370 : internal_fn partial_ifn = IFN_LAST;
11480 256370 : if (loop_lens)
11481 : {
11482 0 : opt_machine_mode new_ovmode
11483 0 : = get_len_load_store_mode (vmode, true, &partial_ifn);
11484 0 : new_vmode = new_ovmode.require ();
11485 0 : unsigned factor
11486 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
11487 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11488 : vec_num, vectype, i, factor, true);
11489 : }
11490 256370 : else if (final_mask)
11491 : {
11492 682 : if (!can_vec_mask_load_store_p (vmode,
11493 682 : TYPE_MODE
11494 : (TREE_TYPE (final_mask)),
11495 : true, &partial_ifn))
11496 0 : gcc_unreachable ();
11497 : }
11498 :
11499 256370 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11500 : {
11501 0 : if (!final_len)
11502 : {
11503 : /* Pass VF value to 'len' argument of
11504 : MASK_LEN_LOAD if LOOP_LENS is invalid. */
11505 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11506 : }
11507 0 : if (!final_mask)
11508 : {
11509 : /* Pass all ones value to 'mask' argument of
11510 : MASK_LEN_LOAD if final_mask is invalid. */
11511 0 : mask_vectype = truth_type_for (vectype);
11512 0 : final_mask = build_minus_one_cst (mask_vectype);
11513 : }
11514 : }
11515 256370 : if (final_len)
11516 : {
11517 0 : signed char biasval
11518 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11519 0 : bias = build_int_cst (intQI_type_node, biasval);
11520 : }
11521 :
11522 0 : tree vec_els;
11523 :
11524 0 : if (final_len)
11525 : {
11526 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11527 0 : gcall *call;
11528 :
11529 : /* Need conversion if the vectype is punned by VnQI. */
11530 0 : els_vectype = vectype;
11531 0 : if (vmode != new_vmode)
11532 0 : els_vectype
11533 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11534 : new_vmode);
11535 0 : vec_els = vect_get_mask_load_else (maskload_elsval,
11536 : els_vectype);
11537 :
11538 0 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11539 : {
11540 0 : if (type_mode_padding_p
11541 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11542 0 : need_zeroing = true;
11543 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
11544 : 6, dataref_ptr, ptr,
11545 : final_mask, vec_els,
11546 : final_len, bias);
11547 : }
11548 : else
11549 0 : call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
11550 : dataref_ptr, ptr,
11551 : vec_els, final_len,
11552 : bias);
11553 0 : gimple_call_set_nothrow (call, true);
11554 0 : new_stmt = call;
11555 0 : data_ref = NULL_TREE;
11556 :
11557 : /* Need conversion if it's wrapped with VnQI. */
11558 0 : if (vmode != new_vmode)
11559 : {
11560 0 : tree new_vtype
11561 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11562 : new_vmode);
11563 0 : tree var = vect_get_new_ssa_name (new_vtype,
11564 : vect_simple_var);
11565 0 : gimple_set_lhs (call, var);
11566 0 : vect_finish_stmt_generation (vinfo, stmt_info, call,
11567 : gsi);
11568 0 : tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
11569 0 : new_stmt = gimple_build_assign (vec_dest,
11570 : VIEW_CONVERT_EXPR, op);
11571 : }
11572 : }
11573 256370 : else if (final_mask)
11574 : {
11575 682 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11576 682 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11577 682 : if (type_mode_padding_p
11578 682 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11579 0 : need_zeroing = true;
11580 682 : gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
11581 : dataref_ptr, ptr,
11582 : final_mask,
11583 : vec_els);
11584 682 : gimple_call_set_nothrow (call, true);
11585 682 : new_stmt = call;
11586 682 : data_ref = NULL_TREE;
11587 : }
11588 : else
11589 : {
11590 255688 : tree ltype = vectype;
11591 255688 : tree new_vtype = NULL_TREE;
11592 255688 : unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
11593 255688 : unsigned HOST_WIDE_INT dr_size
11594 255688 : = vect_get_scalar_dr_size (first_dr_info);
11595 255688 : poly_int64 off = 0;
11596 255688 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11597 1442 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
11598 255688 : unsigned int vect_align
11599 255688 : = vect_known_alignment_in_bytes (first_dr_info, vectype,
11600 255688 : off);
11601 : /* Try to use a single smaller load when we are about
11602 : to load excess elements compared to the unrolled
11603 : scalar loop. */
11604 255688 : if (known_gt ((i + 1) * nunits,
11605 : (group_size * vf - gap)))
11606 : {
11607 7009 : poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
11608 7009 : if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
11609 : nunits))
11610 : /* DR will be unused. */
11611 : ltype = NULL_TREE;
11612 2350 : else if (known_ge (vect_align,
11613 : tree_to_poly_uint64
11614 : (TYPE_SIZE_UNIT (vectype))))
11615 : /* Aligned access to excess elements is OK if
11616 : at least one element is accessed in the
11617 : scalar loop. */
11618 : ;
11619 1949 : else if (known_gt (vect_align,
11620 : ((nunits - remain) * dr_size)))
11621 : /* Aligned access to the gap area when there's
11622 : at least one element in it is OK. */
11623 : ;
11624 : else
11625 : {
11626 : /* remain should now be > 0 and < nunits. */
11627 1946 : unsigned num;
11628 1946 : if (known_ne (remain, 0u)
11629 1946 : && constant_multiple_p (nunits, remain, &num))
11630 : {
11631 1470 : tree ptype;
11632 1470 : new_vtype
11633 1470 : = vector_vector_composition_type (vectype, num,
11634 : &ptype);
11635 1470 : if (new_vtype)
11636 1470 : ltype = ptype;
11637 : }
11638 : /* Else use multiple loads or a masked load? */
11639 : /* For loop vectorization we now should have
11640 : an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11641 : set. */
11642 1946 : if (loop_vinfo)
11643 1666 : gcc_assert (new_vtype
11644 : || LOOP_VINFO_PEELING_FOR_GAPS
11645 : (loop_vinfo));
11646 : /* But still reduce the access size to the next
11647 : required power-of-two so peeling a single
11648 : scalar iteration is sufficient. */
11649 1946 : unsigned HOST_WIDE_INT cremain;
11650 1946 : if (remain.is_constant (&cremain))
11651 : {
11652 1946 : unsigned HOST_WIDE_INT cpart_size
11653 1946 : = 1 << ceil_log2 (cremain);
11654 1946 : if (known_gt (nunits, cpart_size)
11655 1946 : && constant_multiple_p (nunits, cpart_size,
11656 : &num))
11657 : {
11658 1482 : tree ptype;
11659 1482 : new_vtype
11660 2964 : = vector_vector_composition_type (vectype,
11661 1482 : num,
11662 : &ptype);
11663 1482 : if (new_vtype)
11664 1482 : ltype = ptype;
11665 : }
11666 : }
11667 : }
11668 : }
11669 255688 : tree offset = (dataref_offset ? dataref_offset
11670 253562 : : build_int_cst (ref_type, 0));
11671 255688 : if (!ltype)
11672 : ;
11673 251029 : else if (ltype != vectype
11674 251029 : && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11675 : {
11676 25 : poly_uint64 gap_offset
11677 25 : = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
11678 25 : - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
11679 25 : tree gapcst = build_int_cstu (ref_type, gap_offset);
11680 25 : offset = size_binop (PLUS_EXPR, offset, gapcst);
11681 : }
11682 251029 : if (ltype)
11683 : {
11684 251029 : data_ref = fold_build2 (MEM_REF, ltype,
11685 : dataref_ptr, offset);
11686 251029 : if (alignment_support_scheme == dr_aligned
11687 251029 : && align >= TYPE_ALIGN_UNIT (ltype))
11688 : ;
11689 : else
11690 164017 : TREE_TYPE (data_ref)
11691 328034 : = build_aligned_type (TREE_TYPE (data_ref),
11692 : align * BITS_PER_UNIT);
11693 : }
11694 251029 : if (!ltype)
11695 4659 : data_ref = build_constructor (vectype, NULL);
11696 251029 : else if (ltype != vectype)
11697 : {
11698 1482 : vect_copy_ref_info (data_ref,
11699 1482 : DR_REF (first_dr_info->dr));
11700 1482 : tree tem = make_ssa_name (ltype);
11701 1482 : new_stmt = gimple_build_assign (tem, data_ref);
11702 1482 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11703 : gsi);
11704 1482 : data_ref = NULL;
11705 1482 : vec<constructor_elt, va_gc> *v;
11706 : /* We've computed 'num' above to statically two
11707 : or via constant_multiple_p. */
11708 1482 : unsigned num
11709 1482 : = (exact_div (tree_to_poly_uint64
11710 1482 : (TYPE_SIZE_UNIT (vectype)),
11711 : tree_to_poly_uint64
11712 1482 : (TYPE_SIZE_UNIT (ltype)))
11713 1482 : .to_constant ());
11714 1482 : vec_alloc (v, num);
11715 1482 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11716 : {
11717 62 : while (--num)
11718 62 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11719 : build_zero_cst (ltype));
11720 25 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11721 : }
11722 : else
11723 : {
11724 1457 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11725 1457 : while (--num)
11726 3274 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11727 : build_zero_cst (ltype));
11728 : }
11729 1482 : gcc_assert (new_vtype != NULL_TREE);
11730 1482 : if (new_vtype == vectype)
11731 1450 : new_stmt
11732 1450 : = gimple_build_assign (vec_dest,
11733 : build_constructor (vectype, v));
11734 : else
11735 : {
11736 32 : tree new_vname = make_ssa_name (new_vtype);
11737 32 : new_stmt
11738 32 : = gimple_build_assign (new_vname,
11739 : build_constructor (new_vtype,
11740 : v));
11741 32 : vect_finish_stmt_generation (vinfo, stmt_info,
11742 : new_stmt, gsi);
11743 32 : new_stmt
11744 32 : = gimple_build_assign (vec_dest,
11745 : build1 (VIEW_CONVERT_EXPR,
11746 : vectype, new_vname));
11747 : }
11748 : }
11749 : }
11750 : break;
11751 : }
11752 0 : case dr_explicit_realign:
11753 0 : {
11754 0 : if (costing_p)
11755 : break;
11756 0 : tree ptr, bump;
11757 :
11758 0 : tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11759 :
11760 0 : if (compute_in_loop)
11761 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
11762 : gsi, &realignment_token,
11763 : dr_explicit_realign,
11764 : dataref_ptr, NULL);
11765 :
11766 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11767 0 : ptr = copy_ssa_name (dataref_ptr);
11768 : else
11769 0 : ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
11770 : // For explicit realign the target alignment should be
11771 : // known at compile time.
11772 0 : unsigned HOST_WIDE_INT align
11773 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11774 0 : new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
11775 : build_int_cst
11776 0 : (TREE_TYPE (dataref_ptr),
11777 0 : -(HOST_WIDE_INT) align));
11778 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11779 0 : data_ref = build2 (MEM_REF, vectype,
11780 : ptr, build_int_cst (ref_type, 0));
11781 0 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11782 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11783 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11784 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11785 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11786 0 : gimple_move_vops (new_stmt, stmt_info->stmt);
11787 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11788 0 : msq = new_temp;
11789 :
11790 0 : bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
11791 0 : bump = size_binop (MINUS_EXPR, bump, size_one_node);
11792 0 : ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11793 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
11794 0 : build_int_cst (TREE_TYPE (ptr),
11795 0 : -(HOST_WIDE_INT) align));
11796 0 : if (TREE_CODE (ptr) == SSA_NAME)
11797 0 : ptr = copy_ssa_name (ptr, new_stmt);
11798 : else
11799 0 : ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
11800 0 : gimple_assign_set_lhs (new_stmt, ptr);
11801 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11802 0 : data_ref = build2 (MEM_REF, vectype,
11803 : ptr, build_int_cst (ref_type, 0));
11804 0 : break;
11805 : }
11806 0 : case dr_explicit_realign_optimized:
11807 0 : {
11808 0 : if (costing_p)
11809 : break;
11810 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11811 0 : new_temp = copy_ssa_name (dataref_ptr);
11812 : else
11813 0 : new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
11814 : // We should only be doing this if we know the target
11815 : // alignment at compile time.
11816 0 : unsigned HOST_WIDE_INT align
11817 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11818 0 : new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
11819 0 : build_int_cst (TREE_TYPE (dataref_ptr),
11820 0 : -(HOST_WIDE_INT) align));
11821 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11822 0 : data_ref = build2 (MEM_REF, vectype, new_temp,
11823 : build_int_cst (ref_type, 0));
11824 0 : break;
11825 : }
11826 0 : default:
11827 0 : gcc_unreachable ();
11828 : }
11829 :
11830 : /* One common place to cost the above vect load for different
11831 : alignment support schemes. */
11832 958858 : if (costing_p)
11833 : {
11834 : /* For the prologue cost for realign,
11835 : we only need to count it once for the whole group. */
11836 702488 : bool first_stmt_info_p = first_stmt_info == stmt_info;
11837 702488 : bool add_realign_cost = first_stmt_info_p && i == 0;
11838 702488 : if (memory_access_type == VMAT_CONTIGUOUS
11839 702488 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11840 : {
11841 : /* Leave realign cases alone to keep them simple. */
11842 702488 : if (alignment_support_scheme == dr_explicit_realign_optimized
11843 : || alignment_support_scheme == dr_explicit_realign)
11844 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
11845 : alignment_support_scheme, misalignment,
11846 : add_realign_cost, &inside_cost,
11847 : &prologue_cost, cost_vec, cost_vec,
11848 : true);
11849 : else
11850 702488 : n_adjacent_loads++;
11851 : }
11852 : }
11853 : else
11854 : {
11855 256370 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11856 : /* DATA_REF is null if we've already built the statement. */
11857 256370 : if (data_ref)
11858 : {
11859 254206 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11860 254206 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11861 : }
11862 :
11863 512740 : new_temp = (need_zeroing
11864 256370 : ? make_ssa_name (vectype)
11865 256370 : : make_ssa_name (vec_dest, new_stmt));
11866 256370 : gimple_set_lhs (new_stmt, new_temp);
11867 256370 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11868 :
11869 : /* If we need to explicitly zero inactive elements emit a
11870 : VEC_COND_EXPR that does so. */
11871 256370 : if (need_zeroing)
11872 : {
11873 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11874 : vectype);
11875 :
11876 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11877 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11878 : final_mask, new_temp, vec_els);
11879 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11880 : gsi);
11881 0 : new_temp = new_temp2;
11882 : }
11883 : }
11884 :
11885 : /* 3. Handle explicit realignment if necessary/supported.
11886 : Create in loop:
11887 : vec_dest = realign_load (msq, lsq, realignment_token) */
11888 958858 : if (!costing_p
11889 256370 : && (alignment_support_scheme == dr_explicit_realign_optimized
11890 : || alignment_support_scheme == dr_explicit_realign))
11891 : {
11892 0 : lsq = gimple_assign_lhs (new_stmt);
11893 0 : if (!realignment_token)
11894 0 : realignment_token = dataref_ptr;
11895 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11896 0 : new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
11897 : lsq, realignment_token);
11898 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11899 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11900 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11901 :
11902 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
11903 : {
11904 0 : gcc_assert (phi);
11905 0 : if (i == vec_num - 1)
11906 0 : add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
11907 : UNKNOWN_LOCATION);
11908 : msq = lsq;
11909 : }
11910 : }
11911 :
11912 958858 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11913 : {
11914 5933 : if (costing_p)
11915 4491 : inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
11916 : slp_node, 0, vect_body);
11917 : else
11918 : {
11919 1442 : tree perm_mask = perm_mask_for_reverse (vectype);
11920 1442 : new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
11921 : perm_mask, stmt_info, gsi);
11922 1442 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
11923 : }
11924 : }
11925 :
11926 : /* Collect vector loads and later create their permutation in
11927 : vect_transform_slp_perm_load. */
11928 958858 : if (!costing_p && (grouped_load || ls.slp_perm))
11929 74626 : dr_chain.quick_push (new_temp);
11930 :
11931 : /* Store vector loads in the corresponding SLP_NODE. */
11932 256370 : if (!costing_p && !ls.slp_perm)
11933 181744 : slp_node->push_vec_def (new_stmt);
11934 :
11935 : /* With SLP permutation we load the gaps as well, without
11936 : we need to skip the gaps after we manage to fully load
11937 : all elements. group_gap_adj is DR_GROUP_SIZE here. */
11938 958858 : group_elt += nunits;
11939 958858 : if (!costing_p
11940 256370 : && maybe_ne (group_gap_adj, 0U)
11941 44928 : && !ls.slp_perm
11942 978744 : && known_eq (group_elt, group_size - group_gap_adj))
11943 : {
11944 16577 : poly_wide_int bump_val
11945 16577 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11946 16577 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11947 0 : bump_val = -bump_val;
11948 16577 : tree bump = wide_int_to_tree (sizetype, bump_val);
11949 16577 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11950 : bump);
11951 16577 : group_elt = 0;
11952 16577 : }
11953 : }
11954 : /* Bump the vector pointer to account for a gap or for excess
11955 : elements loaded for a permuted SLP load. */
11956 595738 : if (!costing_p
11957 164737 : && maybe_ne (group_gap_adj, 0U)
11958 612808 : && ls.slp_perm)
11959 : {
11960 493 : poly_wide_int bump_val
11961 493 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11962 493 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11963 9 : bump_val = -bump_val;
11964 493 : tree bump = wide_int_to_tree (sizetype, bump_val);
11965 493 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11966 493 : }
11967 :
11968 595738 : if (ls.slp_perm)
11969 : {
11970 : /* For SLP we know we've seen all possible uses of dr_chain so
11971 : direct vect_transform_slp_perm_load to DCE the unused parts.
11972 : ??? This is a hack to prevent compile-time issues as seen
11973 : in PR101120 and friends. */
11974 60475 : if (costing_p)
11975 : {
11976 42944 : gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
11977 42944 : if (ls.n_perms != 0)
11978 42429 : inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11979 : slp_node, 0, vect_body);
11980 42944 : if (n_adjacent_loads > 0)
11981 42944 : n_adjacent_loads = ls.n_loads;
11982 : }
11983 : else
11984 : {
11985 17531 : unsigned n_perms2, n_loads2;
11986 17531 : bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
11987 : gsi, vf, false, &n_perms2,
11988 : &n_loads2, true);
11989 17531 : gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
11990 : }
11991 : }
11992 :
11993 595738 : if (costing_p)
11994 : {
11995 431001 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
11996 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
11997 431001 : if (n_adjacent_loads > 0)
11998 431001 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
11999 : alignment_support_scheme, misalignment, false,
12000 : &inside_cost, &prologue_cost, cost_vec, cost_vec,
12001 : true);
12002 431001 : if (dump_enabled_p ())
12003 24459 : dump_printf_loc (MSG_NOTE, vect_location,
12004 : "vect_model_load_cost: inside_cost = %u, "
12005 : "prologue_cost = %u .\n",
12006 : inside_cost, prologue_cost);
12007 431001 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
12008 431001 : slp_node->data = new vect_load_store_data (std::move (ls));
12009 : }
12010 :
12011 595738 : return true;
12012 1336690 : }
12013 :
12014 : /* vectorizable_condition.
12015 :
12016 : Check if STMT_INFO is conditional modify expression that can be vectorized.
12017 : If COST_VEC is passed, calculate costs but don't change anything,
12018 : otherwise, vectorize STMT_INFO: create a vectorized stmt using
12019 : VEC_COND_EXPR to replace it, and insert it at GSI.
12020 :
12021 : When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12022 :
12023 : Return true if STMT_INFO is vectorizable in this way. */
12024 :
12025 : static bool
12026 726882 : vectorizable_condition (vec_info *vinfo,
12027 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12028 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12029 : {
12030 726882 : tree scalar_dest = NULL_TREE;
12031 726882 : tree vec_dest = NULL_TREE;
12032 726882 : tree then_clause, else_clause;
12033 726882 : tree vec_cond_lhs = NULL_TREE;
12034 726882 : tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
12035 726882 : tree vec_compare;
12036 726882 : tree new_temp;
12037 726882 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
12038 726882 : enum vect_def_type dts[4]
12039 : = {vect_unknown_def_type, vect_unknown_def_type,
12040 : vect_unknown_def_type, vect_unknown_def_type};
12041 726882 : enum tree_code code;
12042 726882 : int i;
12043 726882 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12044 726882 : vec<tree> vec_oprnds0 = vNULL;
12045 726882 : vec<tree> vec_oprnds2 = vNULL;
12046 726882 : vec<tree> vec_oprnds3 = vNULL;
12047 726882 : tree vec_cmp_type;
12048 :
12049 726882 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12050 : return false;
12051 :
12052 : /* Is vectorizable conditional operation? */
12053 726882 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12054 415357 : if (!stmt)
12055 : return false;
12056 :
12057 415357 : code = gimple_assign_rhs_code (stmt);
12058 415357 : if (code != COND_EXPR)
12059 : return false;
12060 :
12061 35530 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
12062 35530 : vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
12063 35530 : bool nested_cycle_p = false;
12064 35530 : bool for_reduction = vect_is_reduction (stmt_info);
12065 35530 : if (for_reduction)
12066 : {
12067 614 : if (SLP_TREE_LANES (slp_node) > 1)
12068 : return false;
12069 : /* ??? With a reduction path we do not get at the reduction info from
12070 : every stmt, use the conservative default setting then. */
12071 694 : if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
12072 : {
12073 596 : vect_reduc_info reduc_info
12074 596 : = info_for_reduction (loop_vinfo, slp_node);
12075 596 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
12076 596 : nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12077 : stmt_info);
12078 : }
12079 : }
12080 : else
12081 : {
12082 34916 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12083 : return false;
12084 : }
12085 :
12086 35530 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12087 35530 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12088 :
12089 35530 : int vec_num = vect_get_num_copies (vinfo, slp_node);
12090 :
12091 35530 : slp_tree slp_cond;
12092 35530 : tree cond_expr = gimple_assign_rhs1 (stmt);
12093 35530 : gcc_assert (! COMPARISON_CLASS_P (cond_expr));
12094 35530 : if (TREE_CODE (cond_expr) != SSA_NAME
12095 35530 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond_expr))
12096 35530 : || !vect_is_simple_use (vinfo, slp_node, 0,
12097 : &slp_cond, &dts[0], &vec_cmp_type)
12098 35530 : || !vec_cmp_type
12099 71044 : || !VECTOR_BOOLEAN_TYPE_P (vec_cmp_type))
12100 : return false;
12101 :
12102 35514 : slp_tree then_slp_node, else_slp_node;
12103 35514 : if (!vect_is_simple_use (vinfo, slp_node, 1,
12104 : &then_clause, &then_slp_node, &dts[2], &vectype1))
12105 : return false;
12106 35514 : if (!vect_is_simple_use (vinfo, slp_node, 2,
12107 : &else_clause, &else_slp_node, &dts[3], &vectype2))
12108 : return false;
12109 :
12110 35514 : if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
12111 : return false;
12112 :
12113 35514 : if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
12114 : return false;
12115 :
12116 35514 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
12117 71028 : TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
12118 : return false;
12119 :
12120 : /* For conditional reductions, the "then" value needs to be the candidate
12121 : value calculated by this iteration while the "else" value needs to be
12122 : the result carried over from previous iterations. If the COND_EXPR
12123 : is the other way around, we need to swap it. */
12124 35514 : bool must_invert_cmp_result = false;
12125 35514 : if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
12126 : {
12127 726882 : must_invert_cmp_result = true;
12128 : /* ??? The vectorized operand query below doesn't allow swapping
12129 : this way for SLP. */
12130 : return false;
12131 : /* std::swap (then_clause, else_clause); */
12132 : }
12133 :
12134 35514 : if (cost_vec)
12135 : {
12136 26848 : vect_cost_for_stmt kind = vector_stmt;
12137 26848 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12138 : /* Count one reduction-like operation per vector. */
12139 : kind = vec_to_scalar;
12140 26848 : else if (!expand_vec_cond_expr_p (vectype, vec_cmp_type))
12141 : return false;
12142 :
12143 26834 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
12144 : vec_cmp_type)
12145 26834 : || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
12146 53668 : || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
12147 : {
12148 0 : if (dump_enabled_p ())
12149 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12150 : "incompatible vector types for invariants\n");
12151 : return false;
12152 : }
12153 :
12154 26834 : if (loop_vinfo && for_reduction
12155 447 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12156 : {
12157 68 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12158 : {
12159 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12160 : vectype, OPTIMIZE_FOR_SPEED))
12161 0 : vect_record_loop_len (loop_vinfo,
12162 : &LOOP_VINFO_LENS (loop_vinfo),
12163 : vec_num, vectype, 1);
12164 : else
12165 0 : vect_record_loop_mask (loop_vinfo,
12166 : &LOOP_VINFO_MASKS (loop_vinfo),
12167 : vec_num, vectype, NULL);
12168 : }
12169 : /* Extra inactive lanes should be safe for vect_nested_cycle. */
12170 68 : else if (!nested_cycle_p)
12171 : {
12172 68 : if (dump_enabled_p ())
12173 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12174 : "conditional reduction prevents the use"
12175 : " of partial vectors.\n");
12176 68 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
12177 : }
12178 : }
12179 :
12180 26834 : SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
12181 26834 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
12182 26834 : return true;
12183 : }
12184 :
12185 : /* Transform. */
12186 :
12187 : /* Handle def. */
12188 8666 : scalar_dest = gimple_assign_lhs (stmt);
12189 8666 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12190 8666 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
12191 :
12192 8666 : bool swap_cond_operands = false;
12193 :
12194 : /* See whether another part of the vectorized code applies a loop
12195 : mask to the condition, or to its inverse. */
12196 :
12197 8666 : vec_loop_masks *masks = NULL;
12198 8666 : vec_loop_lens *lens = NULL;
12199 8666 : if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12200 : {
12201 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12202 0 : lens = &LOOP_VINFO_LENS (loop_vinfo);
12203 : }
12204 8666 : else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
12205 : {
12206 3 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12207 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12208 : else
12209 : {
12210 3 : scalar_cond_masked_key cond (cond_expr, 1);
12211 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12212 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12213 : else
12214 : {
12215 3 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
12216 3 : tree_code orig_code = cond.code;
12217 3 : cond.code = invert_tree_comparison (cond.code, honor_nans);
12218 : /* Try the inverse of the current mask. We check if the
12219 : inverse mask is live and if so we generate a negate of
12220 : the current mask such that we still honor NaNs. */
12221 3 : cond.inverted_p = true;
12222 3 : cond.code = orig_code;
12223 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12224 : {
12225 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12226 0 : swap_cond_operands = true;
12227 0 : must_invert_cmp_result = true;
12228 : }
12229 : }
12230 : }
12231 : }
12232 :
12233 : /* Handle cond expr. */
12234 8666 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds2,
12235 : reduction_type != EXTRACT_LAST_REDUCTION, &vec_oprnds3);
12236 :
12237 8666 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12238 0 : vec_else_clause = else_clause;
12239 :
12240 : /* Arguments are ready. Create the new vector stmt. */
12241 20375 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
12242 : {
12243 11709 : vec_then_clause = vec_oprnds2[i];
12244 11709 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12245 11709 : vec_else_clause = vec_oprnds3[i];
12246 :
12247 11709 : if (swap_cond_operands)
12248 0 : std::swap (vec_then_clause, vec_else_clause);
12249 :
12250 11709 : vec_compare = vec_cond_lhs;
12251 :
12252 : /* If we decided to apply a loop mask to the result of the vector
12253 : comparison, AND the comparison with the mask now. Later passes
12254 : should then be able to reuse the AND results between multiple
12255 : vector statements.
12256 :
12257 : For example:
12258 : for (int i = 0; i < 100; ++i)
12259 : x[i] = y[i] ? z[i] : 10;
12260 :
12261 : results in following optimized GIMPLE:
12262 :
12263 : mask__35.8_43 = vect__4.7_41 != { 0, ... };
12264 : vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12265 : _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12266 : vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12267 : vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12268 : vect_iftmp.11_47, { 10, ... }>;
12269 :
12270 : instead of using a masked and unmasked forms of
12271 : vec != { 0, ... } (masked in the MASK_LOAD,
12272 : unmasked in the VEC_COND_EXPR). */
12273 :
12274 : /* Force vec_compare to be an SSA_NAME rather than a comparison,
12275 : in cases where that's necessary. */
12276 :
12277 11709 : tree len = NULL_TREE, bias = NULL_TREE;
12278 11709 : if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
12279 : {
12280 0 : if (!is_gimple_val (vec_compare))
12281 : {
12282 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12283 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12284 : vec_compare);
12285 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12286 0 : vec_compare = vec_compare_name;
12287 : }
12288 :
12289 0 : if (must_invert_cmp_result)
12290 : {
12291 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12292 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12293 : BIT_NOT_EXPR,
12294 : vec_compare);
12295 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12296 0 : vec_compare = vec_compare_name;
12297 : }
12298 :
12299 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12300 : vectype, OPTIMIZE_FOR_SPEED))
12301 : {
12302 0 : if (lens)
12303 : {
12304 : /* ??? Do we really want the adjusted LEN here? Isn't this
12305 : based on number of elements? */
12306 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens,
12307 : vec_num, vectype, i, 1, true);
12308 0 : signed char biasval
12309 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
12310 0 : bias = build_int_cst (intQI_type_node, biasval);
12311 : }
12312 : else
12313 : {
12314 0 : len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
12315 0 : bias = build_int_cst (intQI_type_node, 0);
12316 : }
12317 : }
12318 0 : if (masks)
12319 : {
12320 0 : tree loop_mask
12321 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12322 : vectype, i);
12323 0 : tree tmp2 = make_ssa_name (vec_cmp_type);
12324 0 : gassign *g
12325 0 : = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
12326 : loop_mask);
12327 0 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
12328 0 : vec_compare = tmp2;
12329 : }
12330 : }
12331 :
12332 0 : gimple *new_stmt;
12333 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12334 : {
12335 0 : gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
12336 0 : tree lhs = gimple_get_lhs (old_stmt);
12337 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12338 0 : lhs = copy_ssa_name (lhs);
12339 0 : if (len)
12340 0 : new_stmt = gimple_build_call_internal
12341 0 : (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
12342 : vec_then_clause, len, bias);
12343 : else
12344 0 : new_stmt = gimple_build_call_internal
12345 0 : (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
12346 : vec_then_clause);
12347 0 : gimple_call_set_lhs (new_stmt, lhs);
12348 0 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12349 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12350 : {
12351 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12352 0 : vec_else_clause = lhs;
12353 : }
12354 0 : else if (old_stmt == gsi_stmt (*gsi))
12355 0 : vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
12356 : else
12357 : {
12358 : /* In this case we're moving the definition to later in the
12359 : block. That doesn't matter because the only uses of the
12360 : lhs are in phi statements. */
12361 0 : gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
12362 0 : gsi_remove (&old_gsi, true);
12363 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12364 : }
12365 : }
12366 : else
12367 : {
12368 11709 : new_temp = make_ssa_name (vec_dest);
12369 11709 : new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
12370 : vec_then_clause, vec_else_clause);
12371 11709 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12372 : }
12373 11709 : slp_node->push_vec_def (new_stmt);
12374 : }
12375 :
12376 8666 : vec_oprnds0.release ();
12377 8666 : vec_oprnds2.release ();
12378 8666 : vec_oprnds3.release ();
12379 :
12380 8666 : return true;
12381 : }
12382 :
12383 : /* Helper of vectorizable_comparison.
12384 :
12385 : Check if STMT_INFO is comparison expression CODE that can be vectorized.
12386 : If COST_VEC is passed, calculate costs but don't change anything,
12387 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12388 : it at GSI.
12389 :
12390 : Return true if STMT_INFO is vectorizable in this way. */
12391 :
12392 : static bool
12393 388437 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
12394 : stmt_vec_info stmt_info, tree_code code,
12395 : gimple_stmt_iterator *gsi,
12396 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12397 : {
12398 388437 : tree lhs, rhs1, rhs2;
12399 388437 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12400 388437 : tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
12401 388437 : tree new_temp;
12402 388437 : enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
12403 388437 : poly_uint64 nunits;
12404 388437 : enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12405 388437 : int i;
12406 388437 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12407 388437 : vec<tree> vec_oprnds0 = vNULL;
12408 388437 : vec<tree> vec_oprnds1 = vNULL;
12409 388437 : tree mask_type;
12410 388437 : tree mask = NULL_TREE;
12411 :
12412 388437 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12413 : return false;
12414 :
12415 388437 : if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
12416 : return false;
12417 :
12418 173304 : mask_type = vectype;
12419 173304 : nunits = TYPE_VECTOR_SUBPARTS (vectype);
12420 :
12421 173304 : if (TREE_CODE_CLASS (code) != tcc_comparison)
12422 : return false;
12423 :
12424 171434 : slp_tree slp_rhs1, slp_rhs2;
12425 171434 : if (!vect_is_simple_use (vinfo, slp_node,
12426 : 0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
12427 : return false;
12428 :
12429 171434 : if (!vect_is_simple_use (vinfo, slp_node,
12430 : 1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
12431 : return false;
12432 :
12433 133506 : if (vectype1 && vectype2
12434 248115 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12435 76681 : TYPE_VECTOR_SUBPARTS (vectype2)))
12436 16 : return false;
12437 :
12438 171418 : vectype = vectype1 ? vectype1 : vectype2;
12439 :
12440 : /* Invariant comparison. */
12441 171418 : if (!vectype)
12442 : {
12443 33146 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
12444 33146 : if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
12445 : return false;
12446 : }
12447 138272 : else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
12448 : return false;
12449 :
12450 : /* Can't compare mask and non-mask types. */
12451 133490 : if (vectype1 && vectype2
12452 400780 : && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
12453 : return false;
12454 :
12455 : /* Boolean values may have another representation in vectors
12456 : and therefore we prefer bit operations over comparison for
12457 : them (which also works for scalar masks). We store opcodes
12458 : to use in bitop1 and bitop2. Statement is vectorized as
12459 : BITOP2 (rhs1 BITOP1 rhs2) or
12460 : rhs1 BITOP2 (BITOP1 rhs2)
12461 : depending on bitop1 and bitop2 arity. */
12462 171401 : bool swap_p = false;
12463 171401 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
12464 : {
12465 769 : if (code == GT_EXPR)
12466 : {
12467 : bitop1 = BIT_NOT_EXPR;
12468 : bitop2 = BIT_AND_EXPR;
12469 : }
12470 : else if (code == GE_EXPR)
12471 : {
12472 : bitop1 = BIT_NOT_EXPR;
12473 : bitop2 = BIT_IOR_EXPR;
12474 : }
12475 : else if (code == LT_EXPR)
12476 : {
12477 : bitop1 = BIT_NOT_EXPR;
12478 : bitop2 = BIT_AND_EXPR;
12479 : swap_p = true;
12480 : }
12481 : else if (code == LE_EXPR)
12482 : {
12483 : bitop1 = BIT_NOT_EXPR;
12484 : bitop2 = BIT_IOR_EXPR;
12485 : swap_p = true;
12486 : }
12487 : else
12488 : {
12489 : bitop1 = BIT_XOR_EXPR;
12490 : if (code == EQ_EXPR)
12491 : bitop2 = BIT_NOT_EXPR;
12492 : }
12493 : }
12494 :
12495 171401 : if (cost_vec)
12496 : {
12497 158692 : if (bitop1 == NOP_EXPR)
12498 : {
12499 158070 : if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
12500 : return false;
12501 : }
12502 : else
12503 : {
12504 622 : machine_mode mode = TYPE_MODE (vectype);
12505 622 : optab optab;
12506 :
12507 622 : optab = optab_for_tree_code (bitop1, vectype, optab_default);
12508 622 : if (!optab || !can_implement_p (optab, mode))
12509 : return false;
12510 :
12511 622 : if (bitop2 != NOP_EXPR)
12512 : {
12513 95 : optab = optab_for_tree_code (bitop2, vectype, optab_default);
12514 95 : if (!optab || !can_implement_p (optab, mode))
12515 : return false;
12516 : }
12517 : }
12518 :
12519 : /* Put types on constant and invariant SLP children. */
12520 148345 : if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
12521 148345 : || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
12522 : {
12523 2 : if (dump_enabled_p ())
12524 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12525 : "incompatible vector types for invariants\n");
12526 : return false;
12527 : }
12528 :
12529 148343 : vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
12530 : slp_node, cost_vec);
12531 148343 : return true;
12532 : }
12533 :
12534 : /* Transform. */
12535 :
12536 : /* Handle def. */
12537 12709 : lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
12538 12709 : if (lhs)
12539 12709 : mask = vect_create_destination_var (lhs, mask_type);
12540 :
12541 12709 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds1);
12542 12709 : if (swap_p)
12543 58 : std::swap (vec_oprnds0, vec_oprnds1);
12544 :
12545 : /* Arguments are ready. Create the new vector stmt. */
12546 31920 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
12547 : {
12548 19211 : gimple *new_stmt;
12549 19211 : vec_rhs2 = vec_oprnds1[i];
12550 :
12551 19211 : if (lhs)
12552 19211 : new_temp = make_ssa_name (mask);
12553 : else
12554 0 : new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
12555 19211 : if (bitop1 == NOP_EXPR)
12556 : {
12557 19054 : new_stmt = gimple_build_assign (new_temp, code,
12558 : vec_rhs1, vec_rhs2);
12559 19054 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12560 : }
12561 : else
12562 : {
12563 157 : if (bitop1 == BIT_NOT_EXPR)
12564 84 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
12565 : else
12566 73 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
12567 : vec_rhs2);
12568 157 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12569 157 : if (bitop2 != NOP_EXPR)
12570 : {
12571 84 : tree res = make_ssa_name (mask);
12572 84 : if (bitop2 == BIT_NOT_EXPR)
12573 0 : new_stmt = gimple_build_assign (res, bitop2, new_temp);
12574 : else
12575 84 : new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
12576 : new_temp);
12577 84 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12578 : }
12579 : }
12580 19211 : slp_node->push_vec_def (new_stmt);
12581 : }
12582 :
12583 12709 : vec_oprnds0.release ();
12584 12709 : vec_oprnds1.release ();
12585 :
12586 12709 : return true;
12587 : }
12588 :
12589 : /* vectorizable_comparison.
12590 :
12591 : Check if STMT_INFO is comparison expression that can be vectorized.
12592 : If COST_VEC is passed, calculate costs but don't change anything,
12593 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12594 : it at GSI.
12595 :
12596 : Return true if STMT_INFO is vectorizable in this way. */
12597 :
12598 : static bool
12599 704091 : vectorizable_comparison (vec_info *vinfo,
12600 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12601 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12602 : {
12603 704091 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12604 :
12605 704091 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12606 : return false;
12607 :
12608 704091 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12609 : return false;
12610 :
12611 459644 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12612 385776 : if (!stmt)
12613 : return false;
12614 :
12615 385776 : enum tree_code code = gimple_assign_rhs_code (stmt);
12616 385776 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12617 385776 : if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
12618 : slp_node, cost_vec))
12619 : return false;
12620 :
12621 158391 : if (cost_vec)
12622 145682 : SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
12623 :
12624 : return true;
12625 : }
12626 :
12627 : /* Check to see if the target supports any of the compare and branch optabs for
12628 : vectors with MODE as these would be required when expanding. */
12629 : static bool
12630 65744 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
12631 : {
12632 65744 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12633 65744 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12634 :
12635 : /* The vectorizer only produces vec_cbranch_any_optab directly. So only
12636 : check for support for that or vec_cbranch_any_optab when masked.
12637 : We can't produce vcond_cbranch_any directly from the vectorizer as we
12638 : want to keep gimple_cond as the GIMPLE representation. But we'll fold
12639 : it in expand. For that reason we require a backend to support the
12640 : unconditional vector cbranch optab if they support the conditional one,
12641 : which is just an optimization on the unconditional one. */
12642 65744 : if (masked_loop_p
12643 65744 : && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
12644 : != CODE_FOR_nothing)
12645 : return true;
12646 65744 : else if (len_loop_p
12647 65744 : && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
12648 : != CODE_FOR_nothing)
12649 : return true;
12650 65744 : else if (!masked_loop_p && !len_loop_p
12651 131488 : && direct_optab_handler (vec_cbranch_any_optab, mode)
12652 : != CODE_FOR_nothing)
12653 : return true;
12654 :
12655 : /* The target can implement cbranch to distinguish between boolean vector
12656 : types and data types if they don't have a different mode for both. */
12657 65744 : return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
12658 : }
12659 :
12660 : /* Determine the type to use for early break vectorization's scalar IV. If
12661 : no type is possible return false. */
12662 :
12663 : static bool
12664 2661 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
12665 : {
12666 : /* Check if we have a usable scalar IV type for vectorization. */
12667 2661 : tree iters_vf_type = sizetype;
12668 2661 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
12669 : {
12670 : /* Find the type with the minimum precision we can use
12671 : for the scalar IV. */
12672 2438 : tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
12673 :
12674 : /* Work out how many bits we need to represent the limit. */
12675 2438 : unsigned int min_ni_width
12676 2438 : = vect_min_prec_for_max_niters (loop_vinfo, 1);
12677 :
12678 : /* Check if we're using PFA, if so we need a signed IV and an
12679 : extra bit for the sign. */
12680 2438 : if (TYPE_UNSIGNED (cand_type)
12681 2438 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
12682 4009 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12683 159 : min_ni_width += 1;
12684 :
12685 2438 : if (TYPE_PRECISION (cand_type) >= min_ni_width)
12686 2365 : iters_vf_type = unsigned_type_for (cand_type);
12687 : else
12688 : {
12689 73 : opt_scalar_int_mode cmp_mode_iter;
12690 73 : tree iv_type = NULL_TREE;
12691 357 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
12692 : {
12693 357 : auto cmp_mode = cmp_mode_iter.require ();
12694 357 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
12695 357 : if (cmp_bits >= min_ni_width
12696 357 : && targetm.scalar_mode_supported_p (cmp_mode))
12697 : {
12698 73 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
12699 73 : if (iv_type)
12700 : break;
12701 : }
12702 : }
12703 :
12704 73 : if (!iv_type)
12705 : {
12706 0 : if (dump_enabled_p ())
12707 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12708 : "can't vectorize early exit because the "
12709 : "target doesn't support a scalar type wide "
12710 : "wide enough to hold niters.\n");
12711 0 : return false;
12712 : }
12713 73 : iters_vf_type = iv_type;
12714 : }
12715 : }
12716 :
12717 2661 : LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
12718 2661 : return true;
12719 : }
12720 :
12721 : /* Check to see if the current early break given in STMT_INFO is valid for
12722 : vectorization. */
12723 :
12724 : bool
12725 262638 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
12726 : gimple_stmt_iterator *gsi,
12727 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12728 : {
12729 262638 : if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
12730 : return false;
12731 :
12732 67363 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
12733 : return false;
12734 :
12735 67363 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
12736 : return false;
12737 :
12738 67363 : DUMP_VECT_SCOPE ("vectorizable_early_exit");
12739 :
12740 67363 : auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
12741 :
12742 : /* For SLP we don't want to use the type of the operands of the SLP node, when
12743 : vectorizing using SLP slp_node will be the children of the gcond and we
12744 : want to use the type of the direct children which since the gcond is root
12745 : will be the current node, rather than a child node as vect_is_simple_use
12746 : assumes. */
12747 67363 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12748 67363 : if (!vectype)
12749 : return false;
12750 :
12751 67363 : machine_mode mode = TYPE_MODE (vectype);
12752 67363 : int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
12753 :
12754 67363 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
12755 67363 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
12756 67363 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12757 67363 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12758 :
12759 : /* Now build the new conditional. Pattern gimple_conds get dropped during
12760 : codegen so we must replace the original insn. */
12761 67363 : gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
12762 67363 : gcond *cond_stmt = as_a <gcond *>(orig_stmt);
12763 :
12764 67363 : tree vectype_out = vectype;
12765 67363 : auto bb = gimple_bb (cond_stmt);
12766 67363 : edge exit_true_edge = EDGE_SUCC (bb, 0);
12767 67363 : if (exit_true_edge->flags & EDGE_FALSE_VALUE)
12768 664 : exit_true_edge = EDGE_SUCC (bb, 1);
12769 67363 : gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
12770 :
12771 : /* When vectorizing we assume that if the branch edge is taken that we're
12772 : exiting the loop. This is not however always the case as the compiler will
12773 : rewrite conditions to always be a comparison against 0. To do this it
12774 : sometimes flips the edges. This is fine for scalar, but for vector we
12775 : then have to negate the result of the test, as we're still assuming that if
12776 : you take the branch edge that we found the exit condition. i.e. we need to
12777 : know whether we are generating a `forall` or an `exist` condition. */
12778 134726 : bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12779 67363 : exit_true_edge->dest);
12780 :
12781 : /* See if we support ADDHN and use that for the reduction. */
12782 67363 : internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
12783 67363 : bool addhn_supported_p
12784 67363 : = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
12785 67363 : tree narrow_type = NULL_TREE;
12786 67363 : if (addhn_supported_p)
12787 : {
12788 : /* Calculate the narrowing type for the result. */
12789 0 : auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
12790 0 : auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
12791 0 : tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
12792 0 : tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
12793 0 : narrow_type = truth_type_for (tmp_type);
12794 :
12795 0 : if (!supports_vector_compare_and_branch (loop_vinfo,
12796 0 : TYPE_MODE (narrow_type)))
12797 : {
12798 0 : if (dump_enabled_p ())
12799 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12800 : "can't use ADDHN reduction because cbranch for "
12801 : "the narrowed type is not supported by the "
12802 : "target.\n");
12803 : addhn_supported_p = false;
12804 : }
12805 : }
12806 :
12807 : /* Analyze only. */
12808 67363 : if (cost_vec)
12809 : {
12810 65744 : if (!addhn_supported_p
12811 65744 : && !supports_vector_compare_and_branch (loop_vinfo, mode))
12812 : {
12813 63083 : if (dump_enabled_p ())
12814 597 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12815 : "can't vectorize early exit because the "
12816 : "target doesn't support flag setting vector "
12817 : "comparisons.\n");
12818 : return false;
12819 : }
12820 :
12821 2661 : if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
12822 : slp_node, cost_vec))
12823 : return false;
12824 :
12825 2661 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12826 : {
12827 1571 : if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
12828 : OPTIMIZE_FOR_SPEED))
12829 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
12830 : else
12831 1571 : vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
12832 : }
12833 :
12834 2661 : if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
12835 : return false;
12836 :
12837 : return true;
12838 : }
12839 :
12840 : /* Transform. */
12841 :
12842 1619 : tree new_temp = NULL_TREE;
12843 1619 : gimple *new_stmt = NULL;
12844 :
12845 1619 : if (dump_enabled_p ())
12846 411 : dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
12847 :
12848 : /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
12849 : roots and so by the time we get to them we have already codegened the SLP tree
12850 : and so we shouldn't try to do so again. The arguments have already been
12851 : vectorized. It's not very clean to do this here, But the masking code below is
12852 : complex and this keeps it all in one place to ease fixes and backports. Once we
12853 : drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
12854 :
12855 1619 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
12856 1619 : basic_block cond_bb = gimple_bb (stmt);
12857 1619 : gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
12858 :
12859 1619 : auto_vec<tree> stmts;
12860 1619 : stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
12861 :
12862 : /* If we're comparing against a previous forall we need to negate the results
12863 : before we do the final comparison or reduction. */
12864 1619 : if (flipped)
12865 : {
12866 : /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
12867 : if (any(~mask)) by negating the masks and flipping the branches.
12868 :
12869 : 1. For unmasked loops we simply reduce the ~mask.
12870 : 2. For masked loops we reduce (~mask & loop_mask) which is the same as
12871 : doing (mask & loop_mask) ^ loop_mask. */
12872 319 : for (unsigned i = 0; i < stmts.length (); i++)
12873 : {
12874 191 : tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
12875 191 : auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
12876 191 : vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
12877 : &cond_gsi);
12878 191 : stmts[i] = inv_lhs;
12879 : }
12880 :
12881 128 : EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12882 128 : EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12883 : }
12884 :
12885 : /* Determine if we need to reduce the final value. */
12886 1619 : if (stmts.length () > 1)
12887 : {
12888 : /* We build the reductions in a way to maintain as much parallelism as
12889 : possible. */
12890 147 : auto_vec<tree> workset (stmts.length ());
12891 :
12892 : /* Mask the statements as we queue them up. Normally we loop over
12893 : vec_num, but since we inspect the exact results of vectorization
12894 : we don't need to and instead can just use the stmts themselves. */
12895 147 : if (masked_loop_p)
12896 0 : for (unsigned i = 0; i < stmts.length (); i++)
12897 : {
12898 0 : tree stmt_mask
12899 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12900 : vectype, i);
12901 0 : stmt_mask
12902 0 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
12903 0 : stmts[i], &cond_gsi);
12904 0 : workset.quick_push (stmt_mask);
12905 : }
12906 147 : else if (len_loop_p)
12907 147 : for (unsigned i = 0; i < stmts.length (); i++)
12908 : {
12909 0 : tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
12910 : lens, vec_num,
12911 0 : vectype, stmts[i], i, 1);
12912 :
12913 0 : workset.quick_push (len_mask);
12914 : }
12915 : else
12916 147 : workset.splice (stmts);
12917 :
12918 442 : while (workset.length () > 1)
12919 : {
12920 295 : tree arg0 = workset.pop ();
12921 295 : tree arg1 = workset.pop ();
12922 295 : if (addhn_supported_p && workset.length () == 0)
12923 : {
12924 0 : new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
12925 0 : vectype_out = narrow_type;
12926 0 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12927 0 : gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
12928 0 : gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
12929 : }
12930 : else
12931 : {
12932 295 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12933 295 : new_stmt
12934 295 : = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
12935 : }
12936 295 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
12937 : &cond_gsi);
12938 295 : workset.quick_insert (0, new_temp);
12939 : }
12940 147 : }
12941 : else
12942 : {
12943 1472 : new_temp = stmts[0];
12944 1472 : if (masked_loop_p)
12945 : {
12946 2 : tree mask
12947 2 : = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
12948 2 : new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
12949 : new_temp, &cond_gsi);
12950 : }
12951 1470 : else if (len_loop_p)
12952 0 : new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
12953 : 1, vectype, new_temp, 0, 1);
12954 : }
12955 :
12956 1619 : gcc_assert (new_temp);
12957 :
12958 1619 : tree cst = build_zero_cst (vectype_out);
12959 1619 : gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
12960 1619 : update_stmt (orig_stmt);
12961 :
12962 : /* ??? */
12963 1619 : SLP_TREE_VEC_DEFS (slp_node).truncate (0);
12964 :
12965 1619 : return true;
12966 1619 : }
12967 :
12968 : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
12969 : can handle all live statements in the node. Otherwise return true
12970 : if STMT_INFO is not live or if vectorizable_live_operation can handle it.
12971 : VEC_STMT_P is as for vectorizable_live_operation. */
12972 :
12973 : static bool
12974 1319786 : can_vectorize_live_stmts (vec_info *vinfo,
12975 : slp_tree slp_node, slp_instance slp_node_instance,
12976 : bool vec_stmt_p,
12977 : stmt_vector_for_cost *cost_vec)
12978 : {
12979 1319786 : stmt_vec_info slp_stmt_info;
12980 1319786 : unsigned int i;
12981 2781005 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
12982 : {
12983 1461219 : if (slp_stmt_info
12984 1445045 : && STMT_VINFO_LIVE_P (slp_stmt_info)
12985 1598785 : && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
12986 : slp_node_instance, i,
12987 : vec_stmt_p, cost_vec))
12988 : return false;
12989 : }
12990 :
12991 : return true;
12992 : }
12993 :
12994 : /* Make sure the statement is vectorizable. */
12995 :
12996 : opt_result
12997 2637789 : vect_analyze_stmt (vec_info *vinfo,
12998 : slp_tree node, slp_instance node_instance,
12999 : stmt_vector_for_cost *cost_vec)
13000 : {
13001 2637789 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
13002 2637789 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
13003 2637789 : enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
13004 2637789 : bool ok;
13005 :
13006 2637789 : if (dump_enabled_p ())
13007 101678 : dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
13008 : stmt_info->stmt);
13009 :
13010 4977536 : if (gimple_has_volatile_ops (stmt_info->stmt))
13011 : {
13012 : /* ??? This shouldn't really happen, volatile stmts should
13013 : not end up in the SLP graph. */
13014 0 : return opt_result::failure_at (stmt_info->stmt,
13015 : "not vectorized:"
13016 : " stmt has volatile operands: %G\n",
13017 : stmt_info->stmt);
13018 : }
13019 :
13020 : /* Skip stmts that do not need to be vectorized. */
13021 2637789 : if (!STMT_VINFO_RELEVANT_P (stmt_info)
13022 0 : && !STMT_VINFO_LIVE_P (stmt_info))
13023 : {
13024 0 : if (dump_enabled_p ())
13025 0 : dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
13026 :
13027 : /* ??? This shouldn't really happen, irrelevant stmts should
13028 : not end up in the SLP graph. */
13029 0 : return opt_result::failure_at (stmt_info->stmt,
13030 : "not vectorized:"
13031 : " irrelevant stmt as SLP node %p "
13032 : "representative.\n",
13033 : (void *)node);
13034 : }
13035 :
13036 2637789 : switch (STMT_VINFO_DEF_TYPE (stmt_info))
13037 : {
13038 : case vect_internal_def:
13039 : case vect_condition_def:
13040 : break;
13041 :
13042 84634 : case vect_reduction_def:
13043 84634 : case vect_nested_cycle:
13044 84634 : gcc_assert (!bb_vinfo
13045 : && (relevance == vect_used_in_outer
13046 : || relevance == vect_used_in_outer_by_reduction
13047 : || relevance == vect_used_by_reduction
13048 : || relevance == vect_unused_in_scope
13049 : || relevance == vect_used_only_live));
13050 : break;
13051 :
13052 322 : case vect_double_reduction_def:
13053 322 : gcc_assert (!bb_vinfo && node);
13054 : break;
13055 :
13056 159485 : case vect_induction_def:
13057 159485 : case vect_first_order_recurrence:
13058 159485 : gcc_assert (!bb_vinfo);
13059 : break;
13060 :
13061 0 : case vect_constant_def:
13062 0 : case vect_external_def:
13063 0 : case vect_unknown_def_type:
13064 0 : default:
13065 0 : gcc_unreachable ();
13066 : }
13067 :
13068 2637789 : tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
13069 2637789 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13070 :
13071 2637789 : if (STMT_VINFO_RELEVANT_P (stmt_info))
13072 : {
13073 2637789 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
13074 2637789 : gcc_assert (SLP_TREE_VECTYPE (node)
13075 : || gimple_code (stmt_info->stmt) == GIMPLE_COND
13076 : || (call && gimple_call_lhs (call) == NULL_TREE));
13077 : }
13078 :
13079 2637789 : ok = true;
13080 2637789 : if (bb_vinfo
13081 1515061 : || (STMT_VINFO_RELEVANT_P (stmt_info)
13082 0 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
13083 : /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13084 : -mveclibabi= takes preference over library functions with
13085 : the simd attribute. */
13086 2637789 : ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
13087 2630889 : || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
13088 : cost_vec)
13089 2630412 : || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
13090 2544234 : || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
13091 2120498 : || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
13092 2049931 : || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
13093 1593067 : || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
13094 762475 : || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
13095 718216 : || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
13096 691382 : || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
13097 545700 : || (bb_vinfo
13098 142234 : && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
13099 3122787 : || (is_a <loop_vec_info> (vinfo)
13100 403466 : && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
13101 : stmt_info, node, cost_vec)
13102 402744 : || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
13103 : stmt_info,
13104 : node, node_instance, cost_vec)
13105 320686 : || vectorizable_induction (as_a <loop_vec_info> (vinfo),
13106 : stmt_info, node, cost_vec)
13107 196356 : || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
13108 : stmt_info, node)
13109 195536 : || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13110 : stmt_info, node, cost_vec)
13111 195275 : || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13112 : stmt_info, NULL, node,
13113 : cost_vec))));
13114 :
13115 2637789 : STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
13116 :
13117 2360982 : if (!ok)
13118 276807 : return opt_result::failure_at (stmt_info->stmt,
13119 : "not vectorized:"
13120 : " relevant stmt not supported: %G",
13121 : stmt_info->stmt);
13122 :
13123 : /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13124 : need extra handling, except for vectorizable reductions. */
13125 2360982 : if (!bb_vinfo
13126 1319786 : && (SLP_TREE_TYPE (node) != lc_phi_info_type
13127 820 : || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
13128 1319786 : && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
13129 3680768 : && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
13130 : node, node_instance,
13131 : false, cost_vec))
13132 0 : return opt_result::failure_at (stmt_info->stmt,
13133 : "not vectorized:"
13134 : " live stmt not supported: %G",
13135 : stmt_info->stmt);
13136 :
13137 2360982 : return opt_result::success ();
13138 : }
13139 :
13140 :
13141 : /* Function vect_transform_stmt.
13142 :
13143 : Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13144 :
13145 : void
13146 992283 : vect_transform_stmt (vec_info *vinfo,
13147 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
13148 : slp_tree slp_node, slp_instance slp_node_instance)
13149 : {
13150 992283 : bool done;
13151 :
13152 992283 : gcc_assert (slp_node);
13153 :
13154 992283 : if (stmt_info)
13155 975483 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13156 :
13157 992283 : switch (SLP_TREE_TYPE (slp_node))
13158 : {
13159 23573 : case type_demotion_vec_info_type:
13160 23573 : case type_promotion_vec_info_type:
13161 23573 : case type_conversion_vec_info_type:
13162 23573 : done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
13163 23573 : gcc_assert (done);
13164 : break;
13165 :
13166 16381 : case induc_vec_info_type:
13167 16381 : done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
13168 : stmt_info, slp_node, NULL);
13169 16381 : gcc_assert (done);
13170 : break;
13171 :
13172 8776 : case shift_vec_info_type:
13173 8776 : done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
13174 8776 : gcc_assert (done);
13175 : break;
13176 :
13177 116938 : case op_vec_info_type:
13178 116938 : done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
13179 116938 : gcc_assert (done);
13180 : break;
13181 :
13182 16366 : case assignment_vec_info_type:
13183 16366 : done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
13184 16366 : gcc_assert (done);
13185 : break;
13186 :
13187 169787 : case load_vec_info_type:
13188 169787 : done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
13189 169787 : gcc_assert (done);
13190 : break;
13191 :
13192 555996 : case store_vec_info_type:
13193 555996 : done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
13194 555996 : gcc_assert (done);
13195 : break;
13196 :
13197 8666 : case condition_vec_info_type:
13198 8666 : done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
13199 8666 : gcc_assert (done);
13200 : break;
13201 :
13202 12709 : case comparison_vec_info_type:
13203 12709 : done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
13204 12709 : gcc_assert (done);
13205 : break;
13206 :
13207 4209 : case call_vec_info_type:
13208 4209 : done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
13209 4209 : break;
13210 :
13211 358 : case call_simd_clone_vec_info_type:
13212 358 : done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
13213 : slp_node, NULL);
13214 358 : break;
13215 :
13216 2667 : case reduc_vec_info_type:
13217 2667 : done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
13218 : gsi, slp_node);
13219 2667 : gcc_assert (done);
13220 : break;
13221 :
13222 23775 : case cycle_phi_info_type:
13223 23775 : done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
13224 : slp_node, slp_node_instance);
13225 23775 : gcc_assert (done);
13226 : break;
13227 :
13228 529 : case lc_phi_info_type:
13229 529 : done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
13230 : stmt_info, slp_node);
13231 529 : gcc_assert (done);
13232 : break;
13233 :
13234 45 : case recurr_info_type:
13235 45 : done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13236 : stmt_info, slp_node, NULL);
13237 45 : gcc_assert (done);
13238 : break;
13239 :
13240 14708 : case phi_info_type:
13241 14708 : done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
13242 : stmt_info, slp_node, NULL);
13243 14708 : gcc_assert (done);
13244 : break;
13245 :
13246 0 : case loop_exit_ctrl_vec_info_type:
13247 0 : done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13248 : stmt_info, gsi, slp_node, NULL);
13249 0 : gcc_assert (done);
13250 : break;
13251 :
13252 16800 : case permute_info_type:
13253 16800 : done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
13254 16800 : gcc_assert (done);
13255 : break;
13256 :
13257 0 : default:
13258 0 : if (!STMT_VINFO_LIVE_P (stmt_info))
13259 : {
13260 0 : if (dump_enabled_p ())
13261 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13262 : "stmt not supported.\n");
13263 0 : gcc_unreachable ();
13264 : }
13265 992283 : done = true;
13266 : }
13267 :
13268 992283 : if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
13269 436287 : && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
13270 : {
13271 : /* Handle stmts whose DEF is used outside the loop-nest that is
13272 : being vectorized. */
13273 600190 : for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
13274 : {
13275 69155 : stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
13276 69155 : done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13277 : slp_node_instance, lane,
13278 : true, NULL);
13279 69155 : gcc_assert (done);
13280 : }
13281 : }
13282 992283 : }
13283 :
13284 :
13285 : /* Remove a group of stores (for SLP or interleaving), free their
13286 : stmt_vec_info. */
13287 :
13288 : void
13289 0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
13290 : {
13291 0 : stmt_vec_info next_stmt_info = first_stmt_info;
13292 :
13293 0 : while (next_stmt_info)
13294 : {
13295 0 : stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
13296 0 : next_stmt_info = vect_orig_stmt (next_stmt_info);
13297 : /* Free the attached stmt_vec_info and remove the stmt. */
13298 0 : vinfo->remove_stmt (next_stmt_info);
13299 0 : next_stmt_info = tmp;
13300 : }
13301 0 : }
13302 :
13303 : /* If NUNITS is nonzero, return a vector type that contains NUNITS
13304 : elements of type SCALAR_TYPE, or null if the target doesn't support
13305 : such a type.
13306 :
13307 : If NUNITS is zero, return a vector type that contains elements of
13308 : type SCALAR_TYPE, choosing whichever vector size the target prefers.
13309 :
13310 : If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13311 : for this vectorization region and want to "autodetect" the best choice.
13312 : Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13313 : and we want the new type to be interoperable with it. PREVAILING_MODE
13314 : in this case can be a scalar integer mode or a vector mode; when it
13315 : is a vector mode, the function acts like a tree-level version of
13316 : related_vector_mode. */
13317 :
13318 : tree
13319 32791944 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
13320 : tree scalar_type, poly_uint64 nunits)
13321 : {
13322 32791944 : tree orig_scalar_type = scalar_type;
13323 32791944 : scalar_mode inner_mode;
13324 32791944 : machine_mode simd_mode;
13325 32791944 : tree vectype;
13326 :
13327 32791944 : if ((!INTEGRAL_TYPE_P (scalar_type)
13328 11279333 : && !POINTER_TYPE_P (scalar_type)
13329 2134332 : && !SCALAR_FLOAT_TYPE_P (scalar_type))
13330 43547213 : || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
13331 1610355 : && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
13332 : return NULL_TREE;
13333 :
13334 32264571 : unsigned int nbytes = GET_MODE_SIZE (inner_mode);
13335 :
13336 : /* Interoperability between modes requires one to be a constant multiple
13337 : of the other, so that the number of vectors required for each operation
13338 : is a compile-time constant. */
13339 32264571 : if (prevailing_mode != VOIDmode
13340 31096516 : && !constant_multiple_p (nunits * nbytes,
13341 31096516 : GET_MODE_SIZE (prevailing_mode))
13342 33875253 : && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
13343 1610682 : nunits * nbytes))
13344 : return NULL_TREE;
13345 :
13346 : /* For vector types of elements whose mode precision doesn't
13347 : match their types precision we use a element type of mode
13348 : precision. The vectorization routines will have to make sure
13349 : they support the proper result truncation/extension.
13350 : We also make sure to build vector types with INTEGER_TYPE
13351 : component type only. */
13352 32264571 : if (INTEGRAL_TYPE_P (scalar_type)
13353 53777095 : && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
13354 19927015 : || TREE_CODE (scalar_type) != INTEGER_TYPE))
13355 1803484 : scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
13356 1803484 : TYPE_UNSIGNED (scalar_type));
13357 :
13358 : /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13359 : When the component mode passes the above test simply use a type
13360 : corresponding to that mode. The theory is that any use that
13361 : would cause problems with this will disable vectorization anyway. */
13362 30461087 : else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
13363 : && !INTEGRAL_TYPE_P (scalar_type))
13364 9145001 : scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
13365 :
13366 : /* We can't build a vector type of elements with alignment bigger than
13367 : their size. */
13368 21316086 : else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
13369 411780 : scalar_type = lang_hooks.types.type_for_mode (inner_mode,
13370 205890 : TYPE_UNSIGNED (scalar_type));
13371 :
13372 : /* If we felt back to using the mode fail if there was
13373 : no scalar type for it. */
13374 32264571 : if (scalar_type == NULL_TREE)
13375 : return NULL_TREE;
13376 :
13377 : /* If no prevailing mode was supplied, use the mode the target prefers.
13378 : Otherwise lookup a vector mode based on the prevailing mode. */
13379 32264571 : if (prevailing_mode == VOIDmode)
13380 : {
13381 1168055 : gcc_assert (known_eq (nunits, 0U));
13382 1168055 : simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
13383 1168055 : if (SCALAR_INT_MODE_P (simd_mode))
13384 : {
13385 : /* Traditional behavior is not to take the integer mode
13386 : literally, but simply to use it as a way of determining
13387 : the vector size. It is up to mode_for_vector to decide
13388 : what the TYPE_MODE should be.
13389 :
13390 : Note that nunits == 1 is allowed in order to support single
13391 : element vector types. */
13392 65458 : if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
13393 32729 : || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13394 : return NULL_TREE;
13395 : }
13396 : }
13397 31096516 : else if (SCALAR_INT_MODE_P (prevailing_mode)
13398 31096516 : || !related_vector_mode (prevailing_mode,
13399 31095699 : inner_mode, nunits).exists (&simd_mode))
13400 : {
13401 : /* Fall back to using mode_for_vector, mostly in the hope of being
13402 : able to use an integer mode. */
13403 2310858 : if (known_eq (nunits, 0U)
13404 4616064 : && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
13405 : return NULL_TREE;
13406 :
13407 236655 : if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13408 222595 : return NULL_TREE;
13409 : }
13410 :
13411 29935692 : vectype = build_vector_type_for_mode (scalar_type, simd_mode);
13412 :
13413 : /* In cases where the mode was chosen by mode_for_vector, check that
13414 : the target actually supports the chosen mode, or that it at least
13415 : allows the vector mode to be replaced by a like-sized integer. */
13416 59871384 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
13417 29950051 : && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
13418 : return NULL_TREE;
13419 :
13420 : /* Re-attach the address-space qualifier if we canonicalized the scalar
13421 : type. */
13422 29927508 : if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
13423 5 : return build_qualified_type
13424 5 : (vectype, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type)));
13425 :
13426 : return vectype;
13427 : }
13428 :
13429 : /* Function get_vectype_for_scalar_type.
13430 :
13431 : Returns the vector type corresponding to SCALAR_TYPE as supported
13432 : by the target. If GROUP_SIZE is nonzero and we're performing BB
13433 : vectorization, make sure that the number of elements in the vector
13434 : is no bigger than GROUP_SIZE. */
13435 :
13436 : tree
13437 27936475 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
13438 : unsigned int group_size)
13439 : {
13440 : /* For BB vectorization, we should always have a group size once we've
13441 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13442 : are tentative requests during things like early data reference
13443 : analysis and pattern recognition. */
13444 27936475 : if (is_a <bb_vec_info> (vinfo))
13445 25128924 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
13446 : else
13447 : group_size = 0;
13448 :
13449 27936475 : tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13450 : scalar_type);
13451 27936475 : if (vectype && vinfo->vector_mode == VOIDmode)
13452 1087223 : vinfo->vector_mode = TYPE_MODE (vectype);
13453 :
13454 : /* Register the natural choice of vector type, before the group size
13455 : has been applied. */
13456 0 : if (vectype)
13457 25244636 : vinfo->used_vector_modes.add (TYPE_MODE (vectype));
13458 :
13459 : /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13460 : try again with an explicit number of elements. */
13461 25244636 : if (vectype
13462 25244636 : && group_size
13463 27936475 : && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
13464 : {
13465 : /* Start with the biggest number of units that fits within
13466 : GROUP_SIZE and halve it until we find a valid vector type.
13467 : Usually either the first attempt will succeed or all will
13468 : fail (in the latter case because GROUP_SIZE is too small
13469 : for the target), but it's possible that a target could have
13470 : a hole between supported vector types.
13471 :
13472 : If GROUP_SIZE is not a power of 2, this has the effect of
13473 : trying the largest power of 2 that fits within the group,
13474 : even though the group is not a multiple of that vector size.
13475 : The BB vectorizer will then try to carve up the group into
13476 : smaller pieces. */
13477 3313223 : unsigned int nunits = 1 << floor_log2 (group_size);
13478 3313223 : do
13479 : {
13480 3313223 : vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13481 3313223 : scalar_type, nunits);
13482 3313223 : nunits /= 2;
13483 : }
13484 3313223 : while (nunits > 1 && !vectype);
13485 : }
13486 :
13487 27936475 : return vectype;
13488 : }
13489 :
13490 : /* Return the vector type corresponding to SCALAR_TYPE as supported
13491 : by the target. NODE, if nonnull, is the SLP tree node that will
13492 : use the returned vector type. */
13493 :
13494 : tree
13495 183698 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
13496 : {
13497 183698 : unsigned int group_size = 0;
13498 183698 : if (node)
13499 183698 : group_size = SLP_TREE_LANES (node);
13500 183698 : return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13501 : }
13502 :
13503 : /* Function get_mask_type_for_scalar_type.
13504 :
13505 : Returns the mask type corresponding to a result of comparison
13506 : of vectors of specified SCALAR_TYPE as supported by target.
13507 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
13508 : make sure that the number of elements in the vector is no bigger
13509 : than GROUP_SIZE. */
13510 :
13511 : tree
13512 1216017 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13513 : unsigned int group_size)
13514 : {
13515 1216017 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13516 :
13517 1216017 : if (!vectype)
13518 : return NULL;
13519 :
13520 1195813 : return truth_type_for (vectype);
13521 : }
13522 :
13523 : /* Function get_mask_type_for_scalar_type.
13524 :
13525 : Returns the mask type corresponding to a result of comparison
13526 : of vectors of specified SCALAR_TYPE as supported by target.
13527 : NODE, if nonnull, is the SLP tree node that will use the returned
13528 : vector type. */
13529 :
13530 : tree
13531 19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13532 : slp_tree node)
13533 : {
13534 19 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
13535 :
13536 19 : if (!vectype)
13537 : return NULL;
13538 :
13539 19 : return truth_type_for (vectype);
13540 : }
13541 :
13542 : /* Function get_same_sized_vectype
13543 :
13544 : Returns a vector type corresponding to SCALAR_TYPE of size
13545 : VECTOR_TYPE if supported by the target. */
13546 :
13547 : tree
13548 166933 : get_same_sized_vectype (tree scalar_type, tree vector_type)
13549 : {
13550 166933 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
13551 0 : return truth_type_for (vector_type);
13552 :
13553 166933 : poly_uint64 nunits;
13554 333866 : if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
13555 333866 : GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
13556 : return NULL_TREE;
13557 :
13558 166933 : return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
13559 166933 : scalar_type, nunits);
13560 : }
13561 :
13562 : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13563 : would not change the chosen vector modes. */
13564 :
13565 : bool
13566 1657887 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
13567 : {
13568 1657887 : for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
13569 2729711 : i != vinfo->used_vector_modes.end (); ++i)
13570 1957743 : if (!VECTOR_MODE_P (*i)
13571 5872959 : || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
13572 885919 : return false;
13573 771968 : return true;
13574 : }
13575 :
13576 : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
13577 : change the chosen vector modes for analysis of a loop. */
13578 :
13579 : bool
13580 392425 : vect_chooses_same_modes_p (machine_mode vector_mode,
13581 : machine_mode alt_vector_mode)
13582 : {
13583 63746 : return (VECTOR_MODE_P (vector_mode)
13584 392425 : && VECTOR_MODE_P (alt_vector_mode)
13585 784850 : && (related_vector_mode (vector_mode,
13586 : GET_MODE_INNER (alt_vector_mode))
13587 392425 : == alt_vector_mode)
13588 418795 : && (related_vector_mode (alt_vector_mode,
13589 : GET_MODE_INNER (vector_mode))
13590 13185 : == vector_mode));
13591 : }
13592 :
13593 : /* Function vect_is_simple_use.
13594 :
13595 : Input:
13596 : VINFO - the vect info of the loop or basic block that is being vectorized.
13597 : OPERAND - operand in the loop or bb.
13598 : Output:
13599 : DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13600 : case OPERAND is an SSA_NAME that is defined in the vectorizable region
13601 : DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13602 : the definition could be anywhere in the function
13603 : DT - the type of definition
13604 :
13605 : Returns whether a stmt with OPERAND can be vectorized.
13606 : For loops, supportable operands are constants, loop invariants, and operands
13607 : that are defined by the current iteration of the loop. Unsupportable
13608 : operands are those that are defined by a previous iteration of the loop (as
13609 : is the case in reduction/induction computations).
13610 : For basic blocks, supportable operands are constants and bb invariants.
13611 : For now, operands defined outside the basic block are not supported. */
13612 :
13613 : bool
13614 43443520 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
13615 : stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
13616 : {
13617 43443520 : if (def_stmt_info_out)
13618 41413059 : *def_stmt_info_out = NULL;
13619 43443520 : if (def_stmt_out)
13620 10104298 : *def_stmt_out = NULL;
13621 43443520 : *dt = vect_unknown_def_type;
13622 :
13623 43443520 : if (dump_enabled_p ())
13624 : {
13625 774998 : dump_printf_loc (MSG_NOTE, vect_location,
13626 : "vect_is_simple_use: operand ");
13627 774998 : if (TREE_CODE (operand) == SSA_NAME
13628 774998 : && !SSA_NAME_IS_DEFAULT_DEF (operand))
13629 709892 : dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
13630 : else
13631 65106 : dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
13632 : }
13633 :
13634 43443520 : if (CONSTANT_CLASS_P (operand))
13635 2984450 : *dt = vect_constant_def;
13636 40459070 : else if (is_gimple_min_invariant (operand))
13637 342782 : *dt = vect_external_def;
13638 40116288 : else if (TREE_CODE (operand) != SSA_NAME)
13639 1029 : *dt = vect_unknown_def_type;
13640 40115259 : else if (SSA_NAME_IS_DEFAULT_DEF (operand))
13641 671588 : *dt = vect_external_def;
13642 : else
13643 : {
13644 39443671 : gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
13645 39443671 : stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
13646 39443671 : if (!stmt_vinfo)
13647 873570 : *dt = vect_external_def;
13648 : else
13649 : {
13650 38570101 : stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
13651 38570101 : def_stmt = stmt_vinfo->stmt;
13652 38570101 : *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
13653 38570101 : if (def_stmt_info_out)
13654 36548422 : *def_stmt_info_out = stmt_vinfo;
13655 : }
13656 39443671 : if (def_stmt_out)
13657 9883752 : *def_stmt_out = def_stmt;
13658 : }
13659 :
13660 43443520 : if (dump_enabled_p ())
13661 : {
13662 774998 : dump_printf (MSG_NOTE, ", type of def: ");
13663 774998 : switch (*dt)
13664 : {
13665 0 : case vect_uninitialized_def:
13666 0 : dump_printf (MSG_NOTE, "uninitialized\n");
13667 0 : break;
13668 54190 : case vect_constant_def:
13669 54190 : dump_printf (MSG_NOTE, "constant\n");
13670 54190 : break;
13671 26853 : case vect_external_def:
13672 26853 : dump_printf (MSG_NOTE, "external\n");
13673 26853 : break;
13674 553206 : case vect_internal_def:
13675 553206 : dump_printf (MSG_NOTE, "internal\n");
13676 553206 : break;
13677 109124 : case vect_induction_def:
13678 109124 : dump_printf (MSG_NOTE, "induction\n");
13679 109124 : break;
13680 28252 : case vect_reduction_def:
13681 28252 : dump_printf (MSG_NOTE, "reduction\n");
13682 28252 : break;
13683 482 : case vect_double_reduction_def:
13684 482 : dump_printf (MSG_NOTE, "double reduction\n");
13685 482 : break;
13686 2178 : case vect_nested_cycle:
13687 2178 : dump_printf (MSG_NOTE, "nested cycle\n");
13688 2178 : break;
13689 277 : case vect_first_order_recurrence:
13690 277 : dump_printf (MSG_NOTE, "first order recurrence\n");
13691 277 : break;
13692 0 : case vect_condition_def:
13693 0 : dump_printf (MSG_NOTE, "control flow\n");
13694 0 : break;
13695 436 : case vect_unknown_def_type:
13696 436 : dump_printf (MSG_NOTE, "unknown\n");
13697 436 : break;
13698 : }
13699 : }
13700 :
13701 43443520 : if (*dt == vect_unknown_def_type)
13702 : {
13703 55162 : if (dump_enabled_p ())
13704 436 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13705 : "Unsupported pattern.\n");
13706 : return false;
13707 : }
13708 :
13709 : return true;
13710 : }
13711 :
13712 : /* Function vect_is_simple_use.
13713 :
13714 : Same as vect_is_simple_use but determines the operand by operand
13715 : position OPERAND from either STMT or SLP_NODE, filling in *OP
13716 : and *SLP_DEF (when SLP_NODE is not NULL). */
13717 :
13718 : bool
13719 3709070 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13720 : unsigned operand, tree *op, slp_tree *slp_def,
13721 : enum vect_def_type *dt,
13722 : tree *vectype, stmt_vec_info *def_stmt_info_out)
13723 : {
13724 3709070 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
13725 3709070 : *slp_def = child;
13726 3709070 : *vectype = SLP_TREE_VECTYPE (child);
13727 3709070 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
13728 : {
13729 : /* ??? VEC_PERM nodes might be intermediate and their lane value
13730 : have no representative (nor do we build a VEC_PERM stmt for
13731 : the actual operation). Note for two-operator nodes we set
13732 : a representative but leave scalar stmts empty as we'd only
13733 : have one for a subset of lanes. Ideally no caller would
13734 : require *op for internal defs. */
13735 2021338 : if (SLP_TREE_REPRESENTATIVE (child))
13736 : {
13737 1963900 : *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
13738 1963900 : return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
13739 : }
13740 : else
13741 : {
13742 57438 : gcc_assert (SLP_TREE_PERMUTE_P (child));
13743 57438 : *op = error_mark_node;
13744 57438 : *dt = vect_internal_def;
13745 57438 : if (def_stmt_info_out)
13746 149 : *def_stmt_info_out = NULL;
13747 : return true;
13748 : }
13749 : }
13750 : else
13751 : {
13752 1687732 : if (def_stmt_info_out)
13753 50314 : *def_stmt_info_out = NULL;
13754 1687732 : *op = SLP_TREE_SCALAR_OPS (child)[0];
13755 1687732 : *dt = SLP_TREE_DEF_TYPE (child);
13756 1687732 : return true;
13757 : }
13758 : }
13759 :
13760 : /* Wrapper around vect_is_simple_use that elides the scalar op output. */
13761 :
13762 : bool
13763 278851 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13764 : unsigned operand, slp_tree *slp_def,
13765 : enum vect_def_type *dt, tree *vectype)
13766 : {
13767 278851 : tree op;
13768 278851 : return vect_is_simple_use (vinfo, slp_node, operand, &op, slp_def, dt,
13769 278851 : vectype);
13770 : }
13771 :
13772 : /* If OP is not NULL and is external or constant update its vector
13773 : type with VECTYPE. Returns true if successful or false if not,
13774 : for example when conflicting vector types are present. */
13775 :
13776 : bool
13777 3224702 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
13778 : {
13779 3224702 : if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
13780 : return true;
13781 1113650 : if (SLP_TREE_VECTYPE (op))
13782 24489 : return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
13783 : /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13784 : should be handled by patters. Allow vect_constant_def for now
13785 : as well as the trivial single-lane uniform vect_external_def case
13786 : both of which we code-generate reasonably. */
13787 1089161 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
13788 1708 : && SLP_TREE_DEF_TYPE (op) == vect_external_def
13789 1090394 : && SLP_TREE_LANES (op) > 1)
13790 : return false;
13791 1088984 : SLP_TREE_VECTYPE (op) = vectype;
13792 1088984 : return true;
13793 : }
13794 :
13795 : /* Function supportable_widening_operation
13796 :
13797 : Check whether an operation represented by the code CODE is a
13798 : widening operation that is supported by the target platform in
13799 : vector form (i.e., when operating on arguments of type VECTYPE_IN
13800 : producing a result of type VECTYPE_OUT).
13801 :
13802 : Widening operations we currently support are NOP (CONVERT), FLOAT,
13803 : FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13804 : are supported by the target platform either directly (via vector
13805 : tree-codes), or via target builtins.
13806 :
13807 : When EVENODD_OK then also lane-swizzling operations are considered.
13808 :
13809 : Output:
13810 : - CODE1 and CODE2 are codes of vector operations to be used when
13811 : vectorizing the operation, if available.
13812 : - MULTI_STEP_CVT determines the number of required intermediate steps in
13813 : case of multi-step conversion (like char->short->int - in that case
13814 : MULTI_STEP_CVT will be 1).
13815 : - INTERM_TYPES contains the intermediate type required to perform the
13816 : widening operation (short in the above example). */
13817 :
13818 : bool
13819 520950 : supportable_widening_operation (code_helper code,
13820 : tree vectype_out, tree vectype_in,
13821 : bool evenodd_ok,
13822 : code_helper *code1,
13823 : code_helper *code2,
13824 : int *multi_step_cvt,
13825 : vec<tree> *interm_types)
13826 : {
13827 520950 : machine_mode vec_mode;
13828 520950 : enum insn_code icode1, icode2;
13829 520950 : optab optab1 = unknown_optab, optab2 = unknown_optab;
13830 520950 : tree vectype = vectype_in;
13831 520950 : tree wide_vectype = vectype_out;
13832 520950 : tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
13833 520950 : int i;
13834 520950 : tree prev_type, intermediate_type;
13835 520950 : machine_mode intermediate_mode, prev_mode;
13836 520950 : optab optab3, optab4;
13837 :
13838 520950 : *multi_step_cvt = 0;
13839 :
13840 520950 : switch (code.safe_as_tree_code ())
13841 : {
13842 : case MAX_TREE_CODES:
13843 : /* Don't set c1 and c2 if code is not a tree_code. */
13844 : break;
13845 :
13846 202625 : case WIDEN_MULT_EXPR:
13847 : /* The result of a vectorized widening operation usually requires
13848 : two vectors (because the widened results do not fit into one vector).
13849 : The generated vector results would normally be expected to be
13850 : generated in the same order as in the original scalar computation,
13851 : i.e. if 8 results are generated in each vector iteration, they are
13852 : to be organized as follows:
13853 : vect1: [res1,res2,res3,res4],
13854 : vect2: [res5,res6,res7,res8].
13855 :
13856 : However, in the special case that the result of the widening
13857 : operation is used in a reduction computation only, the order doesn't
13858 : matter (because when vectorizing a reduction we change the order of
13859 : the computation). Some targets can take advantage of this and
13860 : generate more efficient code. For example, targets like Altivec,
13861 : that support widen_mult using a sequence of {mult_even,mult_odd}
13862 : generate the following vectors:
13863 : vect1: [res1,res3,res5,res7],
13864 : vect2: [res2,res4,res6,res8].
13865 :
13866 : When vectorizing outer-loops, we execute the inner-loop sequentially
13867 : (each vectorized inner-loop iteration contributes to VF outer-loop
13868 : iterations in parallel). We therefore don't allow to change the
13869 : order of the computation in the inner-loop during outer-loop
13870 : vectorization. */
13871 : /* TODO: Another case in which order doesn't *really* matter is when we
13872 : widen and then contract again, e.g. (short)((int)x * y >> 8).
13873 : Normally, pack_trunc performs an even/odd permute, whereas the
13874 : repack from an even/odd expansion would be an interleave, which
13875 : would be significantly simpler for e.g. AVX2. */
13876 : /* In any case, in order to avoid duplicating the code below, recurse
13877 : on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13878 : are properly set up for the caller. If we fail, we'll continue with
13879 : a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13880 202625 : if (evenodd_ok
13881 202625 : && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
13882 : vectype_out, vectype_in,
13883 : evenodd_ok, code1,
13884 : code2, multi_step_cvt,
13885 : interm_types))
13886 104977 : return true;
13887 : c1 = VEC_WIDEN_MULT_LO_EXPR;
13888 : c2 = VEC_WIDEN_MULT_HI_EXPR;
13889 : break;
13890 :
13891 : case DOT_PROD_EXPR:
13892 415973 : c1 = DOT_PROD_EXPR;
13893 415973 : c2 = DOT_PROD_EXPR;
13894 : break;
13895 :
13896 0 : case SAD_EXPR:
13897 0 : c1 = SAD_EXPR;
13898 0 : c2 = SAD_EXPR;
13899 0 : break;
13900 :
13901 200540 : case VEC_WIDEN_MULT_EVEN_EXPR:
13902 : /* Support the recursion induced just above. */
13903 200540 : c1 = VEC_WIDEN_MULT_EVEN_EXPR;
13904 200540 : c2 = VEC_WIDEN_MULT_ODD_EXPR;
13905 200540 : break;
13906 :
13907 9740 : case WIDEN_LSHIFT_EXPR:
13908 9740 : c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
13909 9740 : c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
13910 9740 : break;
13911 :
13912 45987 : CASE_CONVERT:
13913 45987 : c1 = VEC_UNPACK_LO_EXPR;
13914 45987 : c2 = VEC_UNPACK_HI_EXPR;
13915 45987 : break;
13916 :
13917 9210 : case FLOAT_EXPR:
13918 9210 : c1 = VEC_UNPACK_FLOAT_LO_EXPR;
13919 9210 : c2 = VEC_UNPACK_FLOAT_HI_EXPR;
13920 9210 : break;
13921 :
13922 127 : case FIX_TRUNC_EXPR:
13923 127 : c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
13924 127 : c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
13925 127 : break;
13926 :
13927 0 : default:
13928 0 : gcc_unreachable ();
13929 : }
13930 :
13931 415973 : if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
13932 : std::swap (c1, c2);
13933 :
13934 415973 : if (code == FIX_TRUNC_EXPR)
13935 : {
13936 : /* The signedness is determined from output operand. */
13937 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
13938 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
13939 : }
13940 741309 : else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
13941 45987 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
13942 7949 : && VECTOR_BOOLEAN_TYPE_P (vectype)
13943 7949 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
13944 363589 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
13945 : {
13946 : /* If the input and result modes are the same, a different optab
13947 : is needed where we pass in the number of units in vectype. */
13948 : optab1 = vec_unpacks_sbool_lo_optab;
13949 : optab2 = vec_unpacks_sbool_hi_optab;
13950 : }
13951 :
13952 415973 : vec_mode = TYPE_MODE (vectype);
13953 415973 : if (widening_fn_p (code))
13954 : {
13955 : /* If this is an internal fn then we must check whether the target
13956 : supports either a low-high split or an even-odd split. */
13957 52721 : internal_fn ifn = as_internal_fn ((combined_fn) code);
13958 :
13959 52721 : internal_fn lo, hi, even, odd;
13960 52721 : lookup_hilo_internal_fn (ifn, &lo, &hi);
13961 52721 : if (BYTES_BIG_ENDIAN)
13962 : std::swap (lo, hi);
13963 52721 : *code1 = as_combined_fn (lo);
13964 52721 : *code2 = as_combined_fn (hi);
13965 52721 : optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
13966 52721 : optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
13967 :
13968 : /* If we don't support low-high, then check for even-odd. */
13969 52721 : if (!optab1
13970 52721 : || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
13971 0 : || !optab2
13972 52721 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
13973 : {
13974 52721 : lookup_evenodd_internal_fn (ifn, &even, &odd);
13975 52721 : *code1 = as_combined_fn (even);
13976 52721 : *code2 = as_combined_fn (odd);
13977 52721 : optab1 = direct_internal_fn_optab (even, {vectype, vectype});
13978 52721 : optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
13979 : }
13980 : }
13981 363252 : else if (code.is_tree_code ())
13982 : {
13983 363252 : if (code == FIX_TRUNC_EXPR)
13984 : {
13985 : /* The signedness is determined from output operand. */
13986 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
13987 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
13988 : }
13989 363125 : else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
13990 45987 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
13991 7949 : && VECTOR_BOOLEAN_TYPE_P (vectype)
13992 7949 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
13993 363589 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
13994 : {
13995 : /* If the input and result modes are the same, a different optab
13996 : is needed where we pass in the number of units in vectype. */
13997 : optab1 = vec_unpacks_sbool_lo_optab;
13998 : optab2 = vec_unpacks_sbool_hi_optab;
13999 : }
14000 : else
14001 : {
14002 362661 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14003 362661 : optab2 = optab_for_tree_code (c2, vectype, optab_default);
14004 : }
14005 363252 : *code1 = c1;
14006 363252 : *code2 = c2;
14007 : }
14008 :
14009 415973 : if (!optab1 || !optab2)
14010 : return false;
14011 :
14012 415973 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14013 415973 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14014 : return false;
14015 :
14016 :
14017 164829 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14018 164829 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14019 : {
14020 153179 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14021 : return true;
14022 : /* For scalar masks we may have different boolean
14023 : vector types having the same QImode. Thus we
14024 : add additional check for elements number. */
14025 4236 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
14026 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14027 : return true;
14028 : }
14029 :
14030 : /* Check if it's a multi-step conversion that can be done using intermediate
14031 : types. */
14032 :
14033 11855 : prev_type = vectype;
14034 11855 : prev_mode = vec_mode;
14035 :
14036 11855 : if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
14037 : return false;
14038 :
14039 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14040 : intermediate steps in promotion sequence. We try
14041 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14042 : not. */
14043 11803 : interm_types->create (MAX_INTERM_CVT_STEPS);
14044 24863 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14045 : {
14046 13060 : intermediate_mode = insn_data[icode1].operand[0].mode;
14047 13060 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14048 4571 : intermediate_type
14049 4571 : = vect_halve_mask_nunits (prev_type, intermediate_mode);
14050 8489 : else if (VECTOR_MODE_P (intermediate_mode))
14051 : {
14052 8489 : tree intermediate_element_type
14053 8489 : = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
14054 8489 : TYPE_UNSIGNED (prev_type));
14055 8489 : intermediate_type
14056 8489 : = build_vector_type_for_mode (intermediate_element_type,
14057 : intermediate_mode);
14058 8489 : }
14059 : else
14060 0 : intermediate_type
14061 0 : = lang_hooks.types.type_for_mode (intermediate_mode,
14062 0 : TYPE_UNSIGNED (prev_type));
14063 :
14064 13060 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14065 4571 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14066 4571 : && intermediate_mode == TYPE_MODE (wide_vectype)
14067 13331 : && SCALAR_INT_MODE_P (intermediate_mode))
14068 : {
14069 : /* If the input and result modes are the same, a different optab
14070 : is needed where we pass in the number of units in vectype. */
14071 : optab3 = vec_unpacks_sbool_lo_optab;
14072 : optab4 = vec_unpacks_sbool_hi_optab;
14073 : }
14074 : else
14075 : {
14076 12789 : optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
14077 12789 : optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
14078 : }
14079 :
14080 13060 : if (!optab3 || !optab4
14081 13060 : || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
14082 13028 : || insn_data[icode1].operand[0].mode != intermediate_mode
14083 13028 : || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
14084 13028 : || insn_data[icode2].operand[0].mode != intermediate_mode
14085 13028 : || ((icode1 = optab_handler (optab3, intermediate_mode))
14086 : == CODE_FOR_nothing)
14087 25723 : || ((icode2 = optab_handler (optab4, intermediate_mode))
14088 : == CODE_FOR_nothing))
14089 : break;
14090 :
14091 12663 : interm_types->quick_push (intermediate_type);
14092 12663 : (*multi_step_cvt)++;
14093 :
14094 12663 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14095 12663 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14096 : {
14097 11472 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14098 : return true;
14099 3701 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
14100 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14101 : return true;
14102 : }
14103 :
14104 1257 : prev_type = intermediate_type;
14105 1257 : prev_mode = intermediate_mode;
14106 : }
14107 :
14108 397 : interm_types->release ();
14109 397 : return false;
14110 : }
14111 :
14112 :
14113 : /* Function supportable_narrowing_operation
14114 :
14115 : Check whether an operation represented by the code CODE is a
14116 : narrowing operation that is supported by the target platform in
14117 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14118 : and producing a result of type VECTYPE_OUT).
14119 :
14120 : Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14121 : and FLOAT. This function checks if these operations are supported by
14122 : the target platform directly via vector tree-codes.
14123 :
14124 : Output:
14125 : - CODE1 is the code of a vector operation to be used when
14126 : vectorizing the operation, if available.
14127 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14128 : case of multi-step conversion (like int->short->char - in that case
14129 : MULTI_STEP_CVT will be 1).
14130 : - INTERM_TYPES contains the intermediate type required to perform the
14131 : narrowing operation (short in the above example). */
14132 :
14133 : bool
14134 44082 : supportable_narrowing_operation (code_helper code,
14135 : tree vectype_out, tree vectype_in,
14136 : code_helper *code1, int *multi_step_cvt,
14137 : vec<tree> *interm_types)
14138 : {
14139 44082 : machine_mode vec_mode;
14140 44082 : enum insn_code icode1;
14141 44082 : optab optab1, interm_optab;
14142 44082 : tree vectype = vectype_in;
14143 44082 : tree narrow_vectype = vectype_out;
14144 44082 : enum tree_code c1;
14145 44082 : tree intermediate_type, prev_type;
14146 44082 : machine_mode intermediate_mode, prev_mode;
14147 44082 : int i;
14148 44082 : unsigned HOST_WIDE_INT n_elts;
14149 44082 : bool uns;
14150 :
14151 44082 : if (!code.is_tree_code ())
14152 : return false;
14153 :
14154 44082 : *multi_step_cvt = 0;
14155 44082 : switch ((tree_code) code)
14156 : {
14157 42574 : CASE_CONVERT:
14158 42574 : c1 = VEC_PACK_TRUNC_EXPR;
14159 42574 : if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
14160 11559 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14161 11559 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
14162 5258 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
14163 47832 : && n_elts < BITS_PER_UNIT)
14164 : optab1 = vec_pack_sbool_trunc_optab;
14165 : else
14166 40093 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14167 : break;
14168 :
14169 570 : case FIX_TRUNC_EXPR:
14170 570 : c1 = VEC_PACK_FIX_TRUNC_EXPR;
14171 : /* The signedness is determined from output operand. */
14172 570 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14173 570 : break;
14174 :
14175 938 : case FLOAT_EXPR:
14176 938 : c1 = VEC_PACK_FLOAT_EXPR;
14177 938 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14178 938 : break;
14179 :
14180 0 : default:
14181 0 : gcc_unreachable ();
14182 : }
14183 :
14184 44082 : if (!optab1)
14185 : return false;
14186 :
14187 44082 : vec_mode = TYPE_MODE (vectype);
14188 44082 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
14189 : return false;
14190 :
14191 38318 : *code1 = c1;
14192 :
14193 38318 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14194 : {
14195 23858 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14196 : return true;
14197 : /* For scalar masks we may have different boolean
14198 : vector types having the same QImode. Thus we
14199 : add additional check for elements number. */
14200 5799 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
14201 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14202 : return true;
14203 : }
14204 :
14205 14611 : if (code == FLOAT_EXPR)
14206 : return false;
14207 :
14208 : /* Check if it's a multi-step conversion that can be done using intermediate
14209 : types. */
14210 14611 : prev_mode = vec_mode;
14211 14611 : prev_type = vectype;
14212 14611 : if (code == FIX_TRUNC_EXPR)
14213 94 : uns = TYPE_UNSIGNED (vectype_out);
14214 : else
14215 14517 : uns = TYPE_UNSIGNED (vectype);
14216 :
14217 : /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14218 : conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14219 : costly than signed. */
14220 14611 : if (code == FIX_TRUNC_EXPR && uns)
14221 : {
14222 28 : enum insn_code icode2;
14223 :
14224 28 : intermediate_type
14225 28 : = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
14226 28 : interm_optab
14227 28 : = optab_for_tree_code (c1, intermediate_type, optab_default);
14228 28 : if (interm_optab != unknown_optab
14229 28 : && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
14230 28 : && insn_data[icode1].operand[0].mode
14231 28 : == insn_data[icode2].operand[0].mode)
14232 : {
14233 : uns = false;
14234 : optab1 = interm_optab;
14235 : icode1 = icode2;
14236 : }
14237 : }
14238 :
14239 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14240 : intermediate steps in promotion sequence. We try
14241 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14242 14611 : interm_types->create (MAX_INTERM_CVT_STEPS);
14243 31304 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14244 : {
14245 16693 : intermediate_mode = insn_data[icode1].operand[0].mode;
14246 16693 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14247 6997 : intermediate_type
14248 6997 : = vect_double_mask_nunits (prev_type, intermediate_mode);
14249 : else
14250 9696 : intermediate_type
14251 9696 : = lang_hooks.types.type_for_mode (intermediate_mode, uns);
14252 16693 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14253 6997 : && VECTOR_BOOLEAN_TYPE_P (prev_type)
14254 6997 : && SCALAR_INT_MODE_P (prev_mode)
14255 3128 : && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
14256 19821 : && n_elts < BITS_PER_UNIT)
14257 : interm_optab = vec_pack_sbool_trunc_optab;
14258 : else
14259 16341 : interm_optab
14260 16341 : = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
14261 : optab_default);
14262 352 : if (!interm_optab
14263 16693 : || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
14264 16693 : || insn_data[icode1].operand[0].mode != intermediate_mode
14265 33034 : || ((icode1 = optab_handler (interm_optab, intermediate_mode))
14266 : == CODE_FOR_nothing))
14267 : break;
14268 :
14269 15778 : interm_types->quick_push (intermediate_type);
14270 15778 : (*multi_step_cvt)++;
14271 :
14272 15778 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14273 : {
14274 13696 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14275 : return true;
14276 4925 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
14277 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14278 : return true;
14279 : }
14280 :
14281 2082 : prev_mode = intermediate_mode;
14282 2082 : prev_type = intermediate_type;
14283 2082 : optab1 = interm_optab;
14284 : }
14285 :
14286 915 : interm_types->release ();
14287 915 : return false;
14288 : }
14289 :
14290 : /* Function supportable_indirect_convert_operation
14291 :
14292 : Check whether an operation represented by the code CODE is single or multi
14293 : operations that are supported by the target platform in
14294 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14295 : producing a result of type VECTYPE_OUT).
14296 :
14297 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14298 : This function checks if these operations are supported
14299 : by the target platform directly (via vector tree-codes).
14300 :
14301 : Output:
14302 : - converts contains some pairs to perform the convert operation,
14303 : the pair's first is the intermediate type, and its second is the code of
14304 : a vector operation to be used when converting the operation from the
14305 : previous type to the intermediate type. */
14306 : bool
14307 92758 : supportable_indirect_convert_operation (code_helper code,
14308 : tree vectype_out,
14309 : tree vectype_in,
14310 : vec<std::pair<tree, tree_code> > &converts,
14311 : slp_tree slp_op0)
14312 : {
14313 92758 : bool found_mode = false;
14314 92758 : scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
14315 92758 : scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
14316 92758 : tree_code code1, code2;
14317 :
14318 92758 : tree cvt_type = NULL_TREE;
14319 92758 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
14320 :
14321 92758 : if (supportable_convert_operation ((tree_code) code,
14322 : vectype_out,
14323 : vectype_in))
14324 : {
14325 85417 : converts.safe_push (std::make_pair (vectype_out, (tree_code) code));
14326 85417 : return true;
14327 : }
14328 :
14329 : /* For conversions between mask types where the destination has
14330 : a data mode attempt a vcond_mask conversion. */
14331 7341 : if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
14332 143 : && VECTOR_BOOLEAN_TYPE_P (vectype_out)
14333 7484 : && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
14334 : {
14335 138 : tree scalar_datatype
14336 138 : = build_nonstandard_integer_type (element_precision (vectype_out), 0);
14337 138 : tree datatype_out = build_vector_type_for_mode (scalar_datatype,
14338 138 : TYPE_MODE (vectype_out));
14339 138 : if (expand_vec_cond_expr_p (datatype_out, vectype_in))
14340 : {
14341 2 : converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
14342 2 : return true;
14343 : }
14344 : }
14345 :
14346 : /* For conversions between float and integer types try whether
14347 : we can use intermediate signed integer types to support the
14348 : conversion. */
14349 14678 : if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
14350 7339 : && (code == FLOAT_EXPR
14351 4844 : || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
14352 : {
14353 1284 : bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
14354 642 : bool float_expr_p = code == FLOAT_EXPR;
14355 642 : unsigned short target_size;
14356 642 : scalar_mode intermediate_mode;
14357 642 : if (demotion)
14358 : {
14359 99 : intermediate_mode = lhs_mode;
14360 99 : target_size = GET_MODE_SIZE (rhs_mode);
14361 : }
14362 : else
14363 : {
14364 543 : target_size = GET_MODE_SIZE (lhs_mode);
14365 543 : if (!int_mode_for_size
14366 543 : (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
14367 142 : return false;
14368 : }
14369 642 : code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
14370 575 : code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
14371 642 : opt_scalar_mode mode_iter;
14372 1862 : FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
14373 : {
14374 1862 : intermediate_mode = mode_iter.require ();
14375 :
14376 3724 : if (GET_MODE_SIZE (intermediate_mode) > target_size)
14377 : break;
14378 :
14379 1415 : scalar_mode cvt_mode;
14380 1415 : if (!int_mode_for_size
14381 1415 : (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
14382 : break;
14383 :
14384 1385 : cvt_type = build_nonstandard_integer_type
14385 1385 : (GET_MODE_BITSIZE (cvt_mode), 0);
14386 :
14387 : /* Check if the intermediate type can hold OP0's range.
14388 : When converting from float to integer this is not necessary
14389 : because values that do not fit the (smaller) target type are
14390 : unspecified anyway. */
14391 1385 : if (demotion && float_expr_p)
14392 : {
14393 23 : wide_int op_min_value, op_max_value;
14394 : /* For vector form, it looks like slp_op0 doesn't have RANGE_INFO.
14395 : In the future, if it is supported, changes may need to be made
14396 : to this part, such as checking the RANGE of each element
14397 : in the vector. */
14398 23 : if (slp_op0)
14399 : {
14400 13 : tree def;
14401 : /* ??? Merge ranges in case of more than one lane. */
14402 13 : if (SLP_TREE_LANES (slp_op0) != 1
14403 0 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
14404 13 : || !vect_get_range_info (def,
14405 : &op_min_value, &op_max_value))
14406 : break;
14407 : }
14408 : else
14409 : break;
14410 :
14411 0 : if (cvt_type == NULL_TREE
14412 0 : || (wi::min_precision (op_max_value, SIGNED)
14413 0 : > TYPE_PRECISION (cvt_type))
14414 0 : || (wi::min_precision (op_min_value, SIGNED)
14415 0 : > TYPE_PRECISION (cvt_type)))
14416 0 : continue;
14417 23 : }
14418 :
14419 1362 : cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
14420 : cvt_type,
14421 : nelts);
14422 : /* This should only happened for SLP as long as loop vectorizer
14423 : only supports same-sized vector. */
14424 2582 : if (cvt_type == NULL_TREE
14425 2724 : || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
14426 1362 : || !supportable_convert_operation ((tree_code) code1,
14427 : vectype_out, cvt_type)
14428 1943 : || !supportable_convert_operation ((tree_code) code2,
14429 : cvt_type, vectype_in))
14430 1220 : continue;
14431 :
14432 : found_mode = true;
14433 : break;
14434 : }
14435 :
14436 642 : if (found_mode)
14437 : {
14438 142 : converts.safe_push (std::make_pair (cvt_type, code2));
14439 142 : if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
14440 142 : converts.safe_push (std::make_pair (vectype_out, code1));
14441 : return true;
14442 : }
14443 : }
14444 : return false;
14445 : }
14446 :
14447 : /* Generate and return a vector mask of MASK_TYPE such that
14448 : mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14449 : Add the statements to SEQ. */
14450 :
14451 : tree
14452 0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
14453 : tree end_index, const char *name)
14454 : {
14455 0 : tree cmp_type = TREE_TYPE (start_index);
14456 0 : gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
14457 : cmp_type, mask_type,
14458 : OPTIMIZE_FOR_SPEED));
14459 0 : gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
14460 : start_index, end_index,
14461 : build_zero_cst (mask_type));
14462 0 : tree tmp;
14463 0 : if (name)
14464 0 : tmp = make_temp_ssa_name (mask_type, NULL, name);
14465 : else
14466 0 : tmp = make_ssa_name (mask_type);
14467 0 : gimple_call_set_lhs (call, tmp);
14468 0 : gimple_seq_add_stmt (seq, call);
14469 0 : return tmp;
14470 : }
14471 :
14472 : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14473 : J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14474 :
14475 : tree
14476 0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
14477 : tree end_index)
14478 : {
14479 0 : tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
14480 0 : return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
14481 : }
14482 :
14483 : /* Try to compute the vector types required to vectorize STMT_INFO,
14484 : returning true on success and false if vectorization isn't possible.
14485 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
14486 : take sure that the number of elements in the vectors is no bigger
14487 : than GROUP_SIZE.
14488 :
14489 : On success:
14490 :
14491 : - Set *STMT_VECTYPE_OUT to:
14492 : - NULL_TREE if the statement doesn't need to be vectorized;
14493 : - the equivalent of STMT_VINFO_VECTYPE otherwise.
14494 :
14495 : - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14496 : number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14497 : statement does not help to determine the overall number of units. */
14498 :
14499 : opt_result
14500 6181011 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
14501 : tree *stmt_vectype_out,
14502 : tree *nunits_vectype_out,
14503 : unsigned int group_size)
14504 : {
14505 6181011 : gimple *stmt = stmt_info->stmt;
14506 :
14507 : /* For BB vectorization, we should always have a group size once we've
14508 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14509 : are tentative requests during things like early data reference
14510 : analysis and pattern recognition. */
14511 6181011 : if (is_a <bb_vec_info> (vinfo))
14512 4998769 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
14513 : else
14514 : group_size = 0;
14515 :
14516 6181011 : *stmt_vectype_out = NULL_TREE;
14517 6181011 : *nunits_vectype_out = NULL_TREE;
14518 :
14519 6181011 : if (gimple_get_lhs (stmt) == NULL_TREE
14520 : /* Allow vector conditionals through here. */
14521 2610 : && !is_a <gcond *> (stmt)
14522 : /* MASK_STORE and friends have no lhs, but are ok. */
14523 6186211 : && !(is_gimple_call (stmt)
14524 2610 : && gimple_call_internal_p (stmt)
14525 2590 : && internal_store_fn_p (gimple_call_internal_fn (stmt))))
14526 : {
14527 20 : if (is_a <gcall *> (stmt))
14528 : {
14529 : /* Ignore calls with no lhs. These must be calls to
14530 : #pragma omp simd functions, and what vectorization factor
14531 : it really needs can't be determined until
14532 : vectorizable_simd_clone_call. */
14533 20 : if (dump_enabled_p ())
14534 18 : dump_printf_loc (MSG_NOTE, vect_location,
14535 : "defer to SIMD clone analysis.\n");
14536 20 : return opt_result::success ();
14537 : }
14538 :
14539 0 : return opt_result::failure_at (stmt,
14540 : "not vectorized: irregular stmt: %G", stmt);
14541 : }
14542 :
14543 6180991 : tree vectype;
14544 6180991 : tree scalar_type = NULL_TREE;
14545 6180991 : if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
14546 : {
14547 1648063 : vectype = STMT_VINFO_VECTYPE (stmt_info);
14548 1648063 : if (dump_enabled_p ())
14549 80982 : dump_printf_loc (MSG_NOTE, vect_location,
14550 : "precomputed vectype: %T\n", vectype);
14551 : }
14552 4532928 : else if (vect_use_mask_type_p (stmt_info))
14553 : {
14554 234547 : unsigned int precision = stmt_info->mask_precision;
14555 234547 : scalar_type = build_nonstandard_integer_type (precision, 1);
14556 234547 : vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
14557 234547 : if (!vectype)
14558 0 : return opt_result::failure_at (stmt, "not vectorized: unsupported"
14559 : " data-type %T\n", scalar_type);
14560 234547 : if (dump_enabled_p ())
14561 4755 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14562 : }
14563 : else
14564 : {
14565 : /* If we got here with a gcond it means that the target had no available vector
14566 : mode for the scalar type. We can't vectorize so abort. */
14567 4298381 : if (is_a <gcond *> (stmt))
14568 0 : return opt_result::failure_at (stmt,
14569 : "not vectorized:"
14570 : " unsupported data-type for gcond %T\n",
14571 : scalar_type);
14572 :
14573 4298381 : if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
14574 1544936 : scalar_type = TREE_TYPE (DR_REF (dr));
14575 : else
14576 2753445 : scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
14577 :
14578 4298381 : if (dump_enabled_p ())
14579 : {
14580 64917 : if (group_size)
14581 8632 : dump_printf_loc (MSG_NOTE, vect_location,
14582 : "get vectype for scalar type (group size %d):"
14583 : " %T\n", group_size, scalar_type);
14584 : else
14585 56285 : dump_printf_loc (MSG_NOTE, vect_location,
14586 : "get vectype for scalar type: %T\n", scalar_type);
14587 : }
14588 4298381 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
14589 4298381 : if (!vectype)
14590 237113 : return opt_result::failure_at (stmt,
14591 : "not vectorized:"
14592 : " unsupported data-type %T\n",
14593 : scalar_type);
14594 :
14595 4061268 : if (dump_enabled_p ())
14596 64655 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14597 : }
14598 :
14599 4376797 : if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
14600 0 : return opt_result::failure_at (stmt,
14601 : "not vectorized: vector stmt in loop:%G",
14602 : stmt);
14603 :
14604 5943878 : *stmt_vectype_out = vectype;
14605 :
14606 : /* Don't try to compute scalar types if the stmt produces a boolean
14607 : vector; use the existing vector type instead. */
14608 5943878 : tree nunits_vectype = vectype;
14609 5943878 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14610 : {
14611 : /* The number of units is set according to the smallest scalar
14612 : type (or the largest vector size, but we only support one
14613 : vector size per vectorization). */
14614 5370985 : scalar_type = vect_get_smallest_scalar_type (stmt_info,
14615 5370985 : TREE_TYPE (vectype));
14616 5370985 : if (!types_compatible_p (scalar_type, TREE_TYPE (vectype)))
14617 : {
14618 1049592 : if (dump_enabled_p ())
14619 8900 : dump_printf_loc (MSG_NOTE, vect_location,
14620 : "get vectype for smallest scalar type: %T\n",
14621 : scalar_type);
14622 1049592 : nunits_vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
14623 : group_size);
14624 1049592 : if (!nunits_vectype)
14625 16 : return opt_result::failure_at
14626 16 : (stmt, "not vectorized: unsupported data-type %T\n",
14627 : scalar_type);
14628 1049576 : if (dump_enabled_p ())
14629 8900 : dump_printf_loc (MSG_NOTE, vect_location, "nunits vectype: %T\n",
14630 : nunits_vectype);
14631 : }
14632 : }
14633 :
14634 5943862 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (nunits_vectype),
14635 5943862 : TYPE_VECTOR_SUBPARTS (*stmt_vectype_out)))
14636 0 : return opt_result::failure_at (stmt,
14637 : "Not vectorized: Incompatible number "
14638 : "of vector subparts between %T and %T\n",
14639 : nunits_vectype, *stmt_vectype_out);
14640 :
14641 5943862 : if (dump_enabled_p ())
14642 : {
14643 150392 : dump_printf_loc (MSG_NOTE, vect_location, "nunits = ");
14644 150392 : dump_dec (MSG_NOTE, TYPE_VECTOR_SUBPARTS (nunits_vectype));
14645 150392 : dump_printf (MSG_NOTE, "\n");
14646 : }
14647 :
14648 5943862 : *nunits_vectype_out = nunits_vectype;
14649 5943862 : return opt_result::success ();
14650 : }
14651 :
14652 : /* Generate and return statement sequence that sets vector length LEN that is:
14653 :
14654 : min_of_start_and_end = min (START_INDEX, END_INDEX);
14655 : left_len = END_INDEX - min_of_start_and_end;
14656 : rhs = min (left_len, LEN_LIMIT);
14657 : LEN = rhs;
14658 :
14659 : Note: the cost of the code generated by this function is modeled
14660 : by vect_estimate_min_profitable_iters, so changes here may need
14661 : corresponding changes there. */
14662 :
14663 : gimple_seq
14664 0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
14665 : {
14666 0 : gimple_seq stmts = NULL;
14667 0 : tree len_type = TREE_TYPE (len);
14668 0 : gcc_assert (TREE_TYPE (start_index) == len_type);
14669 :
14670 0 : tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
14671 0 : tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
14672 0 : tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
14673 0 : gimple* stmt = gimple_build_assign (len, rhs);
14674 0 : gimple_seq_add_stmt (&stmts, stmt);
14675 :
14676 0 : return stmts;
14677 : }
14678 :
|
