Line data Source code
1 : /* Statement Analysis and Transformation for Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "ssa.h"
31 : #include "optabs-tree.h"
32 : #include "insn-config.h"
33 : #include "recog.h" /* FIXME: for insn_data */
34 : #include "cgraph.h"
35 : #include "dumpfile.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "tree-eh.h"
40 : #include "gimplify.h"
41 : #include "gimple-iterator.h"
42 : #include "gimplify-me.h"
43 : #include "tree-cfg.h"
44 : #include "tree-ssa-loop-manip.h"
45 : #include "cfgloop.h"
46 : #include "explow.h"
47 : #include "tree-ssa-loop.h"
48 : #include "tree-scalar-evolution.h"
49 : #include "tree-vectorizer.h"
50 : #include "builtins.h"
51 : #include "internal-fn.h"
52 : #include "tree-vector-builder.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "tree-ssa-loop-niter.h"
56 : #include "gimple-fold.h"
57 : #include "regs.h"
58 : #include "attribs.h"
59 : #include "optabs-libfuncs.h"
60 : #include "tree-dfa.h"
61 : #include "hierarchical_discriminator.h"
62 :
63 : /* For lang_hooks.types.type_for_mode. */
64 : #include "langhooks.h"
65 :
66 : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
67 : bool = false);
68 :
69 : /* Return TRUE iff the given statement is in an inner loop relative to
70 : the loop being vectorized. */
71 : bool
72 6265360 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
73 : {
74 6265360 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
75 6265360 : basic_block bb = gimple_bb (stmt);
76 6265360 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
77 2795629 : class loop* loop;
78 :
79 2795629 : if (!loop_vinfo)
80 : return false;
81 :
82 2795629 : loop = LOOP_VINFO_LOOP (loop_vinfo);
83 :
84 2795629 : return (bb->loop_father == loop->inner);
85 : }
86 :
87 : /* Record the cost of a statement, either by directly informing the
88 : target model or by saving it in a vector for later processing.
89 : Return a preliminary estimate of the statement's cost. */
90 :
91 : unsigned
92 9217511 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
93 : enum vect_cost_for_stmt kind,
94 : stmt_vec_info stmt_info, slp_tree node,
95 : tree vectype, int misalign,
96 : enum vect_cost_model_location where)
97 : {
98 9217511 : if ((kind == vector_load || kind == unaligned_load)
99 1665913 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
100 : kind = vector_gather_load;
101 9217511 : if ((kind == vector_store || kind == unaligned_store)
102 1049251 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
103 9217511 : kind = vector_scatter_store;
104 :
105 9217511 : stmt_info_for_cost si
106 9217511 : = { count, kind, where, stmt_info, node, vectype, misalign };
107 9217511 : body_cost_vec->safe_push (si);
108 :
109 9217511 : return (unsigned)
110 9217511 : (builtin_vectorization_cost (kind, vectype, misalign) * count);
111 : }
112 :
113 : unsigned
114 4259885 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
115 : enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
116 : tree vectype, int misalign,
117 : enum vect_cost_model_location where)
118 : {
119 4259885 : return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
120 4259885 : vectype, misalign, where);
121 : }
122 :
123 : unsigned
124 1800974 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
125 : enum vect_cost_for_stmt kind, slp_tree node,
126 : tree vectype, int misalign,
127 : enum vect_cost_model_location where)
128 : {
129 1800974 : return record_stmt_cost (body_cost_vec, count, kind,
130 : SLP_TREE_REPRESENTATIVE (node), node,
131 1800974 : vectype, misalign, where);
132 : }
133 :
134 : unsigned
135 0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
136 : enum vect_cost_for_stmt kind,
137 : enum vect_cost_model_location where)
138 : {
139 0 : gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
140 : || kind == scalar_stmt);
141 0 : return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
142 0 : NULL_TREE, 0, where);
143 : }
144 :
145 : /* Return a variable of type ELEM_TYPE[NELEMS]. */
146 :
147 : static tree
148 0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
149 : {
150 0 : return create_tmp_var (build_array_type_nelts (elem_type, nelems),
151 0 : "vect_array");
152 : }
153 :
154 : /* ARRAY is an array of vectors created by create_vector_array.
155 : Return an SSA_NAME for the vector in index N. The reference
156 : is part of the vectorization of STMT_INFO and the vector is associated
157 : with scalar destination SCALAR_DEST.
158 : If we need to ensure that inactive elements are set to zero,
159 : NEED_ZEROING is true, MASK contains the loop mask to be used. */
160 :
161 : static tree
162 0 : read_vector_array (vec_info *vinfo,
163 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
164 : tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
165 : bool need_zeroing, tree mask)
166 : {
167 0 : tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
168 0 : gimple *new_stmt;
169 :
170 0 : gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
171 0 : vect_type = TREE_TYPE (TREE_TYPE (array));
172 0 : tmp = vect_create_destination_var (scalar_dest, vect_type);
173 0 : vect = vect_create_destination_var (scalar_dest, vect_type);
174 0 : array_ref = build4 (ARRAY_REF, vect_type, array,
175 0 : build_int_cst (size_type_node, n),
176 : NULL_TREE, NULL_TREE);
177 :
178 0 : new_stmt = gimple_build_assign (tmp, array_ref);
179 0 : tmp_name = make_ssa_name (vect, new_stmt);
180 0 : gimple_assign_set_lhs (new_stmt, tmp_name);
181 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
182 :
183 0 : if (need_zeroing)
184 : {
185 0 : tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
186 : vect_type);
187 0 : vect_name = make_ssa_name (vect, new_stmt);
188 0 : new_stmt
189 0 : = gimple_build_assign (vect_name, VEC_COND_EXPR,
190 : mask, tmp_name, vec_els);
191 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
192 : }
193 : else
194 : vect_name = tmp_name;
195 :
196 0 : return vect_name;
197 : }
198 :
199 : /* ARRAY is an array of vectors created by create_vector_array.
200 : Emit code to store SSA_NAME VECT in index N of the array.
201 : The store is part of the vectorization of STMT_INFO. */
202 :
203 : static void
204 0 : write_vector_array (vec_info *vinfo,
205 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
206 : tree vect, tree array, unsigned HOST_WIDE_INT n)
207 : {
208 0 : tree array_ref;
209 0 : gimple *new_stmt;
210 :
211 0 : array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
212 0 : build_int_cst (size_type_node, n),
213 : NULL_TREE, NULL_TREE);
214 :
215 0 : new_stmt = gimple_build_assign (array_ref, vect);
216 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
217 0 : }
218 :
219 : /* PTR is a pointer to an array of type TYPE. Return a representation
220 : of *PTR. The memory reference replaces those in FIRST_DR
221 : (and its group). */
222 :
223 : static tree
224 0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
225 : {
226 0 : tree mem_ref;
227 :
228 0 : mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
229 : /* Arrays have the same alignment as their type. */
230 0 : set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
231 0 : return mem_ref;
232 : }
233 :
234 : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
235 : Emit the clobber before *GSI. */
236 :
237 : static void
238 15 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
239 : gimple_stmt_iterator *gsi, tree var)
240 : {
241 15 : tree clobber = build_clobber (TREE_TYPE (var));
242 15 : gimple *new_stmt = gimple_build_assign (var, clobber);
243 15 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
244 15 : }
245 :
246 : /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
247 :
248 : /* Function vect_mark_relevant.
249 :
250 : Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
251 :
252 : static void
253 3302833 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
254 : enum vect_relevant relevant, bool live_p)
255 : {
256 3302833 : enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
257 3302833 : bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
258 :
259 3302833 : if (dump_enabled_p ())
260 169900 : dump_printf_loc (MSG_NOTE, vect_location,
261 : "mark relevant %d, live %d: %G", relevant, live_p,
262 : stmt_info->stmt);
263 :
264 : /* If this stmt is an original stmt in a pattern, we might need to mark its
265 : related pattern stmt instead of the original stmt. However, such stmts
266 : may have their own uses that are not in any pattern, in such cases the
267 : stmt itself should be marked. */
268 3302833 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
269 : {
270 : /* This is the last stmt in a sequence that was detected as a
271 : pattern that can potentially be vectorized. Don't mark the stmt
272 : as relevant/live because it's not going to be vectorized.
273 : Instead mark the pattern-stmt that replaces it. */
274 :
275 255678 : if (dump_enabled_p ())
276 2958 : dump_printf_loc (MSG_NOTE, vect_location,
277 : "last stmt in pattern. don't mark"
278 : " relevant/live.\n");
279 :
280 255678 : stmt_vec_info old_stmt_info = stmt_info;
281 255678 : stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
282 255678 : gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
283 255678 : save_relevant = STMT_VINFO_RELEVANT (stmt_info);
284 255678 : save_live_p = STMT_VINFO_LIVE_P (stmt_info);
285 :
286 255678 : if (live_p && relevant == vect_unused_in_scope)
287 : {
288 107 : if (dump_enabled_p ())
289 10 : dump_printf_loc (MSG_NOTE, vect_location,
290 : "vec_stmt_relevant_p: forcing live pattern stmt "
291 : "relevant.\n");
292 : relevant = vect_used_only_live;
293 : }
294 :
295 255678 : if (dump_enabled_p ())
296 2958 : dump_printf_loc (MSG_NOTE, vect_location,
297 : "mark relevant %d, live %d: %G", relevant, live_p,
298 : stmt_info->stmt);
299 : }
300 :
301 3302833 : STMT_VINFO_LIVE_P (stmt_info) |= live_p;
302 3302833 : if (relevant > STMT_VINFO_RELEVANT (stmt_info))
303 2968778 : STMT_VINFO_RELEVANT (stmt_info) = relevant;
304 :
305 3302833 : if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
306 334055 : && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
307 : {
308 333381 : if (dump_enabled_p ())
309 20569 : dump_printf_loc (MSG_NOTE, vect_location,
310 : "already marked relevant/live.\n");
311 : return;
312 : }
313 :
314 2969452 : worklist->safe_push (stmt_info);
315 : }
316 :
317 :
318 : /* Function is_simple_and_all_uses_invariant
319 :
320 : Return true if STMT_INFO is simple and all uses of it are invariant. */
321 :
322 : bool
323 255032 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
324 : loop_vec_info loop_vinfo)
325 : {
326 255032 : tree op;
327 255032 : ssa_op_iter iter;
328 :
329 255032 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
330 197873 : if (!stmt)
331 : return false;
332 :
333 205342 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
334 : {
335 204561 : enum vect_def_type dt = vect_uninitialized_def;
336 :
337 204561 : if (!vect_is_simple_use (op, loop_vinfo, &dt))
338 : {
339 2212 : if (dump_enabled_p ())
340 22 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
341 : "use not simple.\n");
342 197092 : return false;
343 : }
344 :
345 202349 : if (dt != vect_external_def && dt != vect_constant_def)
346 : return false;
347 : }
348 : return true;
349 : }
350 :
351 : /* Function vect_stmt_relevant_p.
352 :
353 : Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
354 : is "relevant for vectorization".
355 :
356 : A stmt is considered "relevant for vectorization" if:
357 : - it has uses outside the loop.
358 : - it has vdefs (it alters memory).
359 : - control stmts in the loop (except for the exit condition).
360 :
361 : CHECKME: what other side effects would the vectorizer allow? */
362 :
363 : static bool
364 5333176 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
365 : enum vect_relevant *relevant, bool *live_p)
366 : {
367 5333176 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
368 5333176 : ssa_op_iter op_iter;
369 5333176 : imm_use_iterator imm_iter;
370 5333176 : use_operand_p use_p;
371 5333176 : def_operand_p def_p;
372 :
373 5333176 : *relevant = vect_unused_in_scope;
374 5333176 : *live_p = false;
375 :
376 : /* cond stmt other than loop exit cond. */
377 5333176 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
378 5333176 : if (is_ctrl_stmt (stmt)
379 633366 : && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
380 5575065 : && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
381 239928 : *relevant = vect_used_in_scope;
382 :
383 : /* changing memory. */
384 5333176 : if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
385 4417057 : if (gimple_vdef (stmt_info->stmt)
386 3783691 : && !gimple_clobber_p (stmt_info->stmt))
387 : {
388 379519 : if (dump_enabled_p ())
389 28830 : dump_printf_loc (MSG_NOTE, vect_location,
390 : "vec_stmt_relevant_p: stmt has vdefs.\n");
391 379519 : *relevant = vect_used_in_scope;
392 379519 : if (! STMT_VINFO_DATA_REF (stmt_info)
393 379519 : && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
394 20 : LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
395 : }
396 :
397 : /* uses outside the loop. */
398 14980056 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
399 : {
400 11580277 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
401 : {
402 7266573 : basic_block bb = gimple_bb (USE_STMT (use_p));
403 7266573 : if (!flow_bb_inside_loop_p (loop, bb))
404 : {
405 271228 : if (is_gimple_debug (USE_STMT (use_p)))
406 1148 : continue;
407 :
408 270080 : if (dump_enabled_p ())
409 6080 : dump_printf_loc (MSG_NOTE, vect_location,
410 : "vec_stmt_relevant_p: used out of loop.\n");
411 :
412 : /* We expect all such uses to be in the loop exit phis
413 : (because of loop closed form) */
414 270080 : gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
415 :
416 270080 : *live_p = true;
417 270080 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
418 : }
419 4313704 : }
420 : }
421 :
422 255034 : if (*live_p && *relevant == vect_unused_in_scope
423 5588208 : && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
424 : {
425 254251 : if (dump_enabled_p ())
426 5950 : dump_printf_loc (MSG_NOTE, vect_location,
427 : "vec_stmt_relevant_p: stmt live but not relevant.\n");
428 254251 : *relevant = vect_used_only_live;
429 : }
430 :
431 5333176 : return (*live_p || *relevant);
432 : }
433 :
434 :
435 : /* Function exist_non_indexing_operands_for_use_p
436 :
437 : USE is one of the uses attached to STMT_INFO. Check if USE is
438 : used in STMT_INFO for anything other than indexing an array. */
439 :
440 : static bool
441 4423073 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
442 : {
443 4423073 : tree operand;
444 :
445 : /* USE corresponds to some operand in STMT. If there is no data
446 : reference in STMT, then any operand that corresponds to USE
447 : is not indexing an array. */
448 4423073 : if (!STMT_VINFO_DATA_REF (stmt_info))
449 : return true;
450 :
451 : /* STMT has a data_ref. FORNOW this means that its of one of
452 : the following forms:
453 : -1- ARRAY_REF = var
454 : -2- var = ARRAY_REF
455 : (This should have been verified in analyze_data_refs).
456 :
457 : 'var' in the second case corresponds to a def, not a use,
458 : so USE cannot correspond to any operands that are not used
459 : for array indexing.
460 :
461 : Therefore, all we need to check is if STMT falls into the
462 : first case, and whether var corresponds to USE. */
463 :
464 1504140 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
465 1487394 : if (!assign || !gimple_assign_copy_p (assign))
466 : {
467 806076 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
468 16746 : if (call && gimple_call_internal_p (call))
469 : {
470 16746 : internal_fn ifn = gimple_call_internal_fn (call);
471 16746 : int mask_index = internal_fn_mask_index (ifn);
472 16746 : if (mask_index >= 0
473 16746 : && use == gimple_call_arg (call, mask_index))
474 : return true;
475 10930 : int els_index = internal_fn_else_index (ifn);
476 10930 : if (els_index >= 0
477 10930 : && use == gimple_call_arg (call, els_index))
478 : return true;
479 9546 : int stored_value_index = internal_fn_stored_value_index (ifn);
480 9546 : if (stored_value_index >= 0
481 9546 : && use == gimple_call_arg (call, stored_value_index))
482 : return true;
483 7396 : if (internal_gather_scatter_fn_p (ifn)
484 7396 : && use == gimple_call_arg (call, 1))
485 : return true;
486 : }
487 796726 : return false;
488 : }
489 :
490 698064 : if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
491 : return false;
492 698064 : operand = gimple_assign_rhs1 (assign);
493 698064 : if (TREE_CODE (operand) != SSA_NAME)
494 : return false;
495 :
496 603364 : if (operand == use)
497 284686 : return true;
498 :
499 : return false;
500 : }
501 :
502 :
503 : /*
504 : Function process_use.
505 :
506 : Inputs:
507 : - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
508 : - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
509 : that defined USE. This is done by calling mark_relevant and passing it
510 : the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
511 : - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
512 : be performed.
513 :
514 : Outputs:
515 : Generally, LIVE_P and RELEVANT are used to define the liveness and
516 : relevance info of the DEF_STMT of this USE:
517 : STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
518 : STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
519 : Exceptions:
520 : - case 1: If USE is used only for address computations (e.g. array indexing),
521 : which does not need to be directly vectorized, then the liveness/relevance
522 : of the respective DEF_STMT is left unchanged.
523 : - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
524 : we skip DEF_STMT cause it had already been processed.
525 : - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
526 : "relevant" will be modified accordingly.
527 :
528 : Return true if everything is as expected. Return false otherwise. */
529 :
530 : static opt_result
531 4481869 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
532 : enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
533 : bool force)
534 : {
535 4481869 : stmt_vec_info dstmt_vinfo;
536 4481869 : enum vect_def_type dt;
537 :
538 : /* case 1: we are only interested in uses that need to be vectorized. Uses
539 : that are used for address computation are not considered relevant. */
540 4481869 : if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
541 1210104 : return opt_result::success ();
542 :
543 3271765 : if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
544 36083 : return opt_result::failure_at (stmt_vinfo->stmt,
545 : "not vectorized:"
546 : " unsupported use in stmt.\n");
547 :
548 3235682 : if (!dstmt_vinfo)
549 611496 : return opt_result::success ();
550 :
551 2624186 : basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
552 2624186 : basic_block bb = gimple_bb (stmt_vinfo->stmt);
553 :
554 : /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
555 : We have to force the stmt live since the epilogue loop needs it to
556 : continue computing the reduction. */
557 2624186 : if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
558 277030 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
559 86090 : && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
560 86090 : && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
561 2710276 : && bb->loop_father == def_bb->loop_father)
562 : {
563 86090 : if (dump_enabled_p ())
564 4005 : dump_printf_loc (MSG_NOTE, vect_location,
565 : "reduc-stmt defining reduc-phi in the same nest.\n");
566 86090 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
567 86090 : return opt_result::success ();
568 : }
569 :
570 : /* case 3a: outer-loop stmt defining an inner-loop stmt:
571 : outer-loop-header-bb:
572 : d = dstmt_vinfo
573 : inner-loop:
574 : stmt # use (d)
575 : outer-loop-tail-bb:
576 : ... */
577 2538096 : if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
578 : {
579 2132 : if (dump_enabled_p ())
580 321 : dump_printf_loc (MSG_NOTE, vect_location,
581 : "outer-loop def-stmt defining inner-loop stmt.\n");
582 :
583 2132 : switch (relevant)
584 : {
585 0 : case vect_unused_in_scope:
586 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
587 : vect_used_in_scope : vect_unused_in_scope;
588 : break;
589 :
590 767 : case vect_used_in_outer_by_reduction:
591 767 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
592 : relevant = vect_used_by_reduction;
593 : break;
594 :
595 1085 : case vect_used_in_outer:
596 1085 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
597 : relevant = vect_used_in_scope;
598 : break;
599 :
600 : case vect_used_in_scope:
601 : break;
602 :
603 0 : default:
604 0 : gcc_unreachable ();
605 : }
606 : }
607 :
608 : /* case 3b: inner-loop stmt defining an outer-loop stmt:
609 : outer-loop-header-bb:
610 : ...
611 : inner-loop:
612 : d = dstmt_vinfo
613 : outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
614 : stmt # use (d) */
615 2535964 : else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
616 : {
617 2070 : if (dump_enabled_p ())
618 626 : dump_printf_loc (MSG_NOTE, vect_location,
619 : "inner-loop def-stmt defining outer-loop stmt.\n");
620 :
621 2070 : switch (relevant)
622 : {
623 0 : case vect_unused_in_scope:
624 2353330 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
625 0 : || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
626 : vect_used_in_outer_by_reduction : vect_unused_in_scope;
627 : break;
628 :
629 : case vect_used_by_reduction:
630 : case vect_used_only_live:
631 : relevant = vect_used_in_outer_by_reduction;
632 : break;
633 :
634 : case vect_used_in_scope:
635 2353330 : relevant = vect_used_in_outer;
636 : break;
637 :
638 0 : default:
639 0 : gcc_unreachable ();
640 : }
641 : }
642 : /* We are also not interested in uses on loop PHI backedges that are
643 : inductions. Otherwise we'll needlessly vectorize the IV increment
644 : and cause hybrid SLP for SLP inductions. */
645 2533894 : else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
646 187732 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
647 2718660 : && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
648 : loop_latch_edge (bb->loop_father))
649 : == use))
650 : {
651 184766 : if (dump_enabled_p ())
652 5051 : dump_printf_loc (MSG_NOTE, vect_location,
653 : "induction value on backedge.\n");
654 184766 : return opt_result::success ();
655 : }
656 :
657 2353330 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
658 2353330 : return opt_result::success ();
659 : }
660 :
661 :
662 : /* Function vect_mark_stmts_to_be_vectorized.
663 :
664 : Not all stmts in the loop need to be vectorized. For example:
665 :
666 : for i...
667 : for j...
668 : 1. T0 = i + j
669 : 2. T1 = a[T0]
670 :
671 : 3. j = j + 1
672 :
673 : Stmt 1 and 3 do not need to be vectorized, because loop control and
674 : addressing of vectorized data-refs are handled differently.
675 :
676 : This pass detects such stmts. */
677 :
678 : opt_result
679 450343 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
680 : {
681 450343 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
682 450343 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
683 450343 : unsigned int nbbs = loop->num_nodes;
684 450343 : gimple_stmt_iterator si;
685 450343 : unsigned int i;
686 450343 : basic_block bb;
687 450343 : bool live_p;
688 450343 : enum vect_relevant relevant;
689 :
690 450343 : DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
691 :
692 450343 : auto_vec<stmt_vec_info, 64> worklist;
693 :
694 : /* 1. Init worklist. */
695 1526775 : for (i = 0; i < nbbs; i++)
696 : {
697 1087643 : bb = bbs[i];
698 2229035 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
699 : {
700 2304916 : if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
701 236339 : continue;
702 916119 : stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
703 916119 : if (dump_enabled_p ())
704 42965 : dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
705 : phi_info->stmt);
706 :
707 916119 : if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
708 : {
709 48285 : if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
710 11066 : return opt_result::failure_at
711 11066 : (*si, "not vectorized: unhandled relevant PHI: %G", *si);
712 37219 : vect_mark_relevant (&worklist, phi_info, relevant, live_p);
713 : }
714 : }
715 8683717 : for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
716 : {
717 7607285 : gimple *stmt = gsi_stmt (si);
718 7607285 : if (is_gimple_debug (stmt))
719 3190083 : continue;
720 4417202 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
721 4417202 : if (dump_enabled_p ())
722 231840 : dump_printf_loc (MSG_NOTE, vect_location,
723 : "init: stmt relevant? %G", stmt);
724 :
725 4417202 : if (gimple_get_lhs (stmt) == NULL_TREE
726 639594 : && !is_a <gcond *> (stmt)
727 4423430 : && !is_a <gcall *> (stmt))
728 145 : return opt_result::failure_at
729 145 : (stmt, "not vectorized: irregular stmt: %G", stmt);
730 :
731 4417057 : if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
732 826194 : vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
733 : }
734 : }
735 :
736 : /* 2. Process_worklist */
737 3275842 : while (worklist.length () > 0)
738 : {
739 2872795 : use_operand_p use_p;
740 2872795 : ssa_op_iter iter;
741 :
742 2872795 : stmt_vec_info stmt_vinfo = worklist.pop ();
743 2872795 : if (dump_enabled_p ())
744 148670 : dump_printf_loc (MSG_NOTE, vect_location,
745 : "worklist: examine stmt: %G", stmt_vinfo->stmt);
746 :
747 : /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
748 : (DEF_STMT) as relevant/irrelevant according to the relevance property
749 : of STMT. */
750 2872795 : relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
751 :
752 : /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
753 : propagated as is to the DEF_STMTs of its USEs.
754 :
755 : One exception is when STMT has been identified as defining a reduction
756 : variable; in this case we set the relevance to vect_used_by_reduction.
757 : This is because we distinguish between two kinds of relevant stmts -
758 : those that are used by a reduction computation, and those that are
759 : (also) used by a regular computation. This allows us later on to
760 : identify stmts that are used solely by a reduction, and therefore the
761 : order of the results that they produce does not have to be kept. */
762 :
763 2872795 : switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
764 : {
765 175145 : case vect_reduction_def:
766 175145 : gcc_assert (relevant != vect_unused_in_scope);
767 175145 : if (relevant != vect_unused_in_scope
768 175145 : && relevant != vect_used_in_scope
769 175145 : && relevant != vect_used_by_reduction
770 175145 : && relevant != vect_used_only_live)
771 0 : return opt_result::failure_at
772 0 : (stmt_vinfo->stmt, "unsupported use of reduction.\n");
773 : break;
774 :
775 2089 : case vect_nested_cycle:
776 2089 : if (relevant != vect_unused_in_scope
777 2089 : && relevant != vect_used_in_outer_by_reduction
778 1494 : && relevant != vect_used_in_outer)
779 2 : return opt_result::failure_at
780 2 : (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
781 : break;
782 :
783 1209 : case vect_double_reduction_def:
784 1209 : if (relevant != vect_unused_in_scope
785 1209 : && relevant != vect_used_by_reduction
786 409 : && relevant != vect_used_only_live)
787 0 : return opt_result::failure_at
788 0 : (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
789 : break;
790 :
791 : default:
792 : break;
793 : }
794 :
795 2872793 : if (is_pattern_stmt_p (stmt_vinfo))
796 : {
797 : /* Pattern statements are not inserted into the code, so
798 : FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
799 : have to scan the RHS or function arguments instead. */
800 654760 : if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
801 : {
802 425434 : enum tree_code rhs_code = gimple_assign_rhs_code (assign);
803 425434 : tree op = gimple_assign_rhs1 (assign);
804 :
805 425434 : i = 1;
806 425434 : if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
807 : {
808 0 : opt_result res
809 0 : = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
810 : loop_vinfo, relevant, &worklist, false);
811 0 : if (!res)
812 0 : return res;
813 0 : res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
814 : loop_vinfo, relevant, &worklist, false);
815 0 : if (!res)
816 0 : return res;
817 425434 : i = 2;
818 : }
819 1226820 : for (; i < gimple_num_ops (assign); i++)
820 : {
821 805380 : op = gimple_op (assign, i);
822 805380 : if (TREE_CODE (op) == SSA_NAME)
823 : {
824 608557 : opt_result res
825 608557 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
826 : &worklist, false);
827 608557 : if (!res)
828 3994 : return res;
829 : }
830 : }
831 : }
832 229326 : else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
833 : {
834 222385 : tree_code rhs_code = gimple_cond_code (cond);
835 222385 : gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
836 222385 : opt_result res
837 222385 : = process_use (stmt_vinfo, gimple_cond_lhs (cond),
838 : loop_vinfo, relevant, &worklist, false);
839 222385 : if (!res)
840 36085 : return res;
841 222385 : res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
842 : loop_vinfo, relevant, &worklist, false);
843 222385 : if (!res)
844 0 : return res;
845 : }
846 6941 : else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
847 : {
848 32395 : for (i = 0; i < gimple_call_num_args (call); i++)
849 : {
850 25454 : tree arg = gimple_call_arg (call, i);
851 25454 : opt_result res
852 25454 : = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
853 : &worklist, false);
854 25454 : if (!res)
855 0 : return res;
856 : }
857 : }
858 : else
859 0 : gcc_unreachable ();
860 : }
861 : else
862 7762366 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
863 : {
864 3344292 : tree op = USE_FROM_PTR (use_p);
865 3344292 : opt_result res
866 3344292 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
867 : &worklist, false);
868 3344292 : if (!res)
869 17992 : return res;
870 : }
871 :
872 2850807 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
873 : {
874 58796 : gather_scatter_info gs_info;
875 58796 : if (!vect_check_gather_scatter (stmt_vinfo,
876 : STMT_VINFO_VECTYPE (stmt_vinfo),
877 : loop_vinfo, &gs_info))
878 0 : gcc_unreachable ();
879 58796 : opt_result res
880 58796 : = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
881 : &worklist, true);
882 58796 : if (!res)
883 : {
884 14097 : if (fatal)
885 14097 : *fatal = false;
886 14097 : return res;
887 : }
888 : }
889 : } /* while worklist */
890 :
891 403047 : return opt_result::success ();
892 450343 : }
893 :
894 : /* Function vect_model_simple_cost.
895 :
896 : Models cost for simple operations, i.e. those that only emit N operations
897 : of the same KIND. */
898 :
899 : static void
900 676967 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
901 : stmt_vector_for_cost *cost_vec,
902 : vect_cost_for_stmt kind = vector_stmt)
903 : {
904 676967 : int inside_cost = 0, prologue_cost = 0;
905 :
906 676967 : gcc_assert (cost_vec != NULL);
907 :
908 676967 : n *= vect_get_num_copies (vinfo, node);
909 :
910 : /* Pass the inside-of-loop statements to the target-specific cost model. */
911 676967 : inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
912 :
913 676967 : if (dump_enabled_p ())
914 35456 : dump_printf_loc (MSG_NOTE, vect_location,
915 : "vect_model_simple_cost: inside_cost = %d, "
916 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
917 676967 : }
918 :
919 :
920 : /* Model cost for type demotion and promotion operations. PWR is
921 : normally zero for single-step promotions and demotions. It will be
922 : one if two-step promotion/demotion is required, and so on. NCOPIES
923 : is the number of vector results (and thus number of instructions)
924 : for the narrowest end of the operation chain. Each additional
925 : step doubles the number of instructions required. If WIDEN_ARITH
926 : is true the stmt is doing widening arithmetic. */
927 :
928 : static void
929 69543 : vect_model_promotion_demotion_cost (slp_tree slp_node,
930 : unsigned int ncopies, int pwr,
931 : stmt_vector_for_cost *cost_vec,
932 : bool widen_arith)
933 : {
934 69543 : int i;
935 69543 : int inside_cost = 0, prologue_cost = 0;
936 :
937 162135 : for (i = 0; i < pwr + 1; i++)
938 : {
939 183279 : inside_cost += record_stmt_cost (cost_vec, ncopies,
940 : widen_arith
941 : ? vector_stmt : vec_promote_demote,
942 : slp_node, 0, vect_body);
943 92592 : ncopies *= 2;
944 : }
945 :
946 69543 : if (dump_enabled_p ())
947 6461 : dump_printf_loc (MSG_NOTE, vect_location,
948 : "vect_model_promotion_demotion_cost: inside_cost = %d, "
949 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
950 69543 : }
951 :
952 : /* Returns true if the current function returns DECL. */
953 :
954 : static bool
955 571257 : cfun_returns (tree decl)
956 : {
957 571257 : edge_iterator ei;
958 571257 : edge e;
959 1124885 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
960 : {
961 1131078 : greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
962 565539 : if (!ret)
963 0 : continue;
964 565539 : if (gimple_return_retval (ret) == decl)
965 : return true;
966 : /* We often end up with an aggregate copy to the result decl,
967 : handle that case as well. First skip intermediate clobbers
968 : though. */
969 : gimple *def = ret;
970 1707175 : do
971 : {
972 3414350 : def = SSA_NAME_DEF_STMT (gimple_vuse (def));
973 : }
974 1707175 : while (gimple_clobber_p (def));
975 554421 : if (is_a <gassign *> (def)
976 63483 : && gimple_assign_lhs (def) == gimple_return_retval (ret)
977 561464 : && gimple_assign_rhs1 (def) == decl)
978 : return true;
979 : }
980 : return false;
981 : }
982 :
983 : /* Calculate cost of DR's memory access. */
984 : void
985 1039110 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
986 : int ncopies, dr_alignment_support alignment_support_scheme,
987 : int misalignment,
988 : unsigned int *inside_cost,
989 : stmt_vector_for_cost *body_cost_vec)
990 : {
991 1039110 : tree vectype
992 1039110 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
993 1039110 : switch (alignment_support_scheme)
994 : {
995 571502 : case dr_aligned:
996 571502 : {
997 571502 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
998 : vector_store, stmt_info, slp_node,
999 : vectype, 0, vect_body);
1000 :
1001 571502 : if (dump_enabled_p ())
1002 15282 : dump_printf_loc (MSG_NOTE, vect_location,
1003 : "vect_model_store_cost: aligned.\n");
1004 : break;
1005 : }
1006 :
1007 467608 : case dr_unaligned_supported:
1008 467608 : {
1009 : /* Here, we assign an additional cost for the unaligned store. */
1010 467608 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1011 : unaligned_store, stmt_info, slp_node,
1012 : vectype, misalignment, vect_body);
1013 467608 : if (dump_enabled_p ())
1014 13696 : dump_printf_loc (MSG_NOTE, vect_location,
1015 : "vect_model_store_cost: unaligned supported by "
1016 : "hardware.\n");
1017 : break;
1018 : }
1019 :
1020 0 : case dr_unaligned_unsupported:
1021 0 : {
1022 0 : *inside_cost = VECT_MAX_COST;
1023 :
1024 0 : if (dump_enabled_p ())
1025 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1026 : "vect_model_store_cost: unsupported access.\n");
1027 : break;
1028 : }
1029 :
1030 0 : default:
1031 0 : gcc_unreachable ();
1032 : }
1033 1039110 : }
1034 :
1035 : /* Calculate cost of DR's memory access. */
1036 : void
1037 986271 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
1038 : int ncopies, dr_alignment_support alignment_support_scheme,
1039 : int misalignment,
1040 : bool add_realign_cost, unsigned int *inside_cost,
1041 : unsigned int *prologue_cost,
1042 : stmt_vector_for_cost *prologue_cost_vec,
1043 : stmt_vector_for_cost *body_cost_vec,
1044 : bool record_prologue_costs)
1045 : {
1046 986271 : tree vectype
1047 986271 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
1048 986271 : switch (alignment_support_scheme)
1049 : {
1050 566980 : case dr_aligned:
1051 566980 : {
1052 566980 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1053 : stmt_info, slp_node, vectype,
1054 : 0, vect_body);
1055 :
1056 566980 : if (dump_enabled_p ())
1057 19920 : dump_printf_loc (MSG_NOTE, vect_location,
1058 : "vect_model_load_cost: aligned.\n");
1059 :
1060 : break;
1061 : }
1062 358831 : case dr_unaligned_supported:
1063 358831 : {
1064 : /* Here, we assign an additional cost for the unaligned load. */
1065 358831 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1066 : unaligned_load, stmt_info, slp_node,
1067 : vectype, misalignment, vect_body);
1068 :
1069 358831 : if (dump_enabled_p ())
1070 23912 : dump_printf_loc (MSG_NOTE, vect_location,
1071 : "vect_model_load_cost: unaligned supported by "
1072 : "hardware.\n");
1073 :
1074 : break;
1075 : }
1076 0 : case dr_explicit_realign:
1077 0 : {
1078 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1079 : vector_load, stmt_info, slp_node,
1080 : vectype, 0, vect_body);
1081 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1082 : vec_perm, stmt_info, slp_node,
1083 : vectype, 0, vect_body);
1084 :
1085 : /* FIXME: If the misalignment remains fixed across the iterations of
1086 : the containing loop, the following cost should be added to the
1087 : prologue costs. */
1088 0 : if (targetm.vectorize.builtin_mask_for_load)
1089 0 : *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1090 : stmt_info, slp_node, vectype,
1091 : 0, vect_body);
1092 :
1093 0 : if (dump_enabled_p ())
1094 0 : dump_printf_loc (MSG_NOTE, vect_location,
1095 : "vect_model_load_cost: explicit realign\n");
1096 :
1097 : break;
1098 : }
1099 0 : case dr_explicit_realign_optimized:
1100 0 : {
1101 0 : if (dump_enabled_p ())
1102 0 : dump_printf_loc (MSG_NOTE, vect_location,
1103 : "vect_model_load_cost: unaligned software "
1104 : "pipelined.\n");
1105 :
1106 : /* Unaligned software pipeline has a load of an address, an initial
1107 : load, and possibly a mask operation to "prime" the loop. However,
1108 : if this is an access in a group of loads, which provide grouped
1109 : access, then the above cost should only be considered for one
1110 : access in the group. Inside the loop, there is a load op
1111 : and a realignment op. */
1112 :
1113 0 : if (add_realign_cost && record_prologue_costs)
1114 : {
1115 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1116 : vector_stmt, stmt_info,
1117 : slp_node, vectype,
1118 : 0, vect_prologue);
1119 0 : if (targetm.vectorize.builtin_mask_for_load)
1120 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1121 : vector_stmt, stmt_info,
1122 : slp_node, vectype,
1123 : 0, vect_prologue);
1124 : }
1125 :
1126 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1127 : stmt_info, slp_node, vectype,
1128 : 0, vect_body);
1129 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1130 : stmt_info, slp_node, vectype,
1131 : 0, vect_body);
1132 :
1133 0 : if (dump_enabled_p ())
1134 0 : dump_printf_loc (MSG_NOTE, vect_location,
1135 : "vect_model_load_cost: explicit realign optimized"
1136 : "\n");
1137 :
1138 : break;
1139 : }
1140 :
1141 60460 : case dr_unaligned_unsupported:
1142 60460 : {
1143 60460 : *inside_cost = VECT_MAX_COST;
1144 :
1145 60460 : if (dump_enabled_p ())
1146 116 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1147 : "vect_model_load_cost: unsupported access.\n");
1148 : break;
1149 : }
1150 :
1151 0 : default:
1152 0 : gcc_unreachable ();
1153 : }
1154 986271 : }
1155 :
1156 : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1157 : the loop preheader for the vectorized stmt STMT_VINFO. */
1158 :
1159 : static void
1160 6693 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
1161 : gimple_stmt_iterator *gsi)
1162 : {
1163 6693 : if (gsi)
1164 3415 : vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
1165 : else
1166 3278 : vinfo->insert_on_entry (stmt_vinfo, new_stmt);
1167 :
1168 6693 : if (dump_enabled_p ())
1169 1858 : dump_printf_loc (MSG_NOTE, vect_location,
1170 : "created new init_stmt: %G", new_stmt);
1171 6693 : }
1172 :
1173 : /* Function vect_init_vector.
1174 :
1175 : Insert a new stmt (INIT_STMT) that initializes a new variable of type
1176 : TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1177 : vector type a vector with all elements equal to VAL is created first.
1178 : Place the initialization at GSI if it is not NULL. Otherwise, place the
1179 : initialization at the loop preheader.
1180 : Return the DEF of INIT_STMT.
1181 : It will be used in the vectorization of STMT_INFO. */
1182 :
1183 : tree
1184 4975 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
1185 : gimple_stmt_iterator *gsi)
1186 : {
1187 4975 : gimple *init_stmt;
1188 4975 : tree new_temp;
1189 :
1190 : /* We abuse this function to push sth to a SSA name with initial 'val'. */
1191 4975 : if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1192 : {
1193 1318 : gcc_assert (VECTOR_TYPE_P (type));
1194 1318 : if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1195 : {
1196 : /* Scalar boolean value should be transformed into
1197 : all zeros or all ones value before building a vector. */
1198 11 : if (VECTOR_BOOLEAN_TYPE_P (type))
1199 : {
1200 3 : tree true_val = build_all_ones_cst (TREE_TYPE (type));
1201 3 : tree false_val = build_zero_cst (TREE_TYPE (type));
1202 :
1203 3 : if (CONSTANT_CLASS_P (val))
1204 0 : val = integer_zerop (val) ? false_val : true_val;
1205 : else
1206 : {
1207 3 : new_temp = make_ssa_name (TREE_TYPE (type));
1208 3 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1209 : val, true_val, false_val);
1210 3 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1211 3 : val = new_temp;
1212 : }
1213 : }
1214 : else
1215 : {
1216 8 : gimple_seq stmts = NULL;
1217 8 : if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1218 8 : val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
1219 8 : TREE_TYPE (type), val);
1220 : else
1221 : /* ??? Condition vectorization expects us to do
1222 : promotion of invariant/external defs. */
1223 0 : val = gimple_convert (&stmts, TREE_TYPE (type), val);
1224 16 : for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
1225 16 : !gsi_end_p (gsi2); )
1226 : {
1227 8 : init_stmt = gsi_stmt (gsi2);
1228 8 : gsi_remove (&gsi2, false);
1229 8 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1230 : }
1231 : }
1232 : }
1233 1318 : val = build_vector_from_val (type, val);
1234 : }
1235 :
1236 4975 : new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1237 4975 : init_stmt = gimple_build_assign (new_temp, val);
1238 4975 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1239 4975 : return new_temp;
1240 : }
1241 :
1242 :
1243 : /* Get vectorized definitions for OP0 and OP1. */
1244 :
1245 : void
1246 196888 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
1247 : bool op0, vec<tree> *vec_oprnds0,
1248 : bool op1, vec<tree> *vec_oprnds1,
1249 : bool op2, vec<tree> *vec_oprnds2,
1250 : bool op3, vec<tree> *vec_oprnds3)
1251 : {
1252 196888 : if (op0)
1253 195285 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
1254 196888 : if (op1)
1255 144839 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
1256 196888 : if (op2)
1257 9500 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
1258 196888 : if (op3)
1259 0 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
1260 196888 : }
1261 :
1262 : /* Helper function called by vect_finish_replace_stmt and
1263 : vect_finish_stmt_generation. Set the location of the new
1264 : statement and create and return a stmt_vec_info for it. */
1265 :
1266 : static void
1267 1439810 : vect_finish_stmt_generation_1 (vec_info *,
1268 : stmt_vec_info stmt_info, gimple *vec_stmt)
1269 : {
1270 1439810 : if (dump_enabled_p ())
1271 150477 : dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
1272 :
1273 1439810 : if (stmt_info)
1274 : {
1275 1410959 : gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
1276 :
1277 : /* While EH edges will generally prevent vectorization, stmt might
1278 : e.g. be in a must-not-throw region. Ensure newly created stmts
1279 : that could throw are part of the same region. */
1280 1410959 : int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
1281 1410959 : if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
1282 48 : add_stmt_to_eh_lp (vec_stmt, lp_nr);
1283 : }
1284 : else
1285 28851 : gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
1286 1439810 : }
1287 :
1288 : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1289 : which sets the same scalar result as STMT_INFO did. Create and return a
1290 : stmt_vec_info for VEC_STMT. */
1291 :
1292 : void
1293 930 : vect_finish_replace_stmt (vec_info *vinfo,
1294 : stmt_vec_info stmt_info, gimple *vec_stmt)
1295 : {
1296 930 : gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
1297 930 : gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
1298 :
1299 930 : gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
1300 930 : gsi_replace (&gsi, vec_stmt, true);
1301 :
1302 930 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1303 930 : }
1304 :
1305 : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1306 : before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1307 :
1308 : void
1309 1438880 : vect_finish_stmt_generation (vec_info *vinfo,
1310 : stmt_vec_info stmt_info, gimple *vec_stmt,
1311 : gimple_stmt_iterator *gsi)
1312 : {
1313 1438880 : gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
1314 :
1315 1438880 : if (!gsi_end_p (*gsi)
1316 2876806 : && gimple_has_mem_ops (vec_stmt))
1317 : {
1318 1437926 : gimple *at_stmt = gsi_stmt (*gsi);
1319 1437926 : tree vuse = gimple_vuse (at_stmt);
1320 1431736 : if (vuse && TREE_CODE (vuse) == SSA_NAME)
1321 : {
1322 1279853 : tree vdef = gimple_vdef (at_stmt);
1323 1279853 : gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1324 1279853 : gimple_set_modified (vec_stmt, true);
1325 : /* If we have an SSA vuse and insert a store, update virtual
1326 : SSA form to avoid triggering the renamer. Do so only
1327 : if we can easily see all uses - which is what almost always
1328 : happens with the way vectorized stmts are inserted. */
1329 759448 : if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1330 2039265 : && ((is_gimple_assign (vec_stmt)
1331 758575 : && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1332 64578 : || (is_gimple_call (vec_stmt)
1333 837 : && (!(gimple_call_flags (vec_stmt)
1334 837 : & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
1335 3 : || (gimple_call_lhs (vec_stmt)
1336 3 : && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
1337 : {
1338 695668 : tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1339 695668 : gimple_set_vdef (vec_stmt, new_vdef);
1340 695668 : SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1341 : }
1342 : }
1343 : }
1344 1438880 : gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1345 1438880 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1346 1438880 : }
1347 :
1348 : /* We want to vectorize a call to combined function CFN with function
1349 : decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1350 : as the types of all inputs. Check whether this is possible using
1351 : an internal function, returning its code if so or IFN_LAST if not. */
1352 :
1353 : static internal_fn
1354 17137 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
1355 : tree vectype_out, tree vectype_in)
1356 : {
1357 17137 : internal_fn ifn;
1358 17137 : if (internal_fn_p (cfn))
1359 14504 : ifn = as_internal_fn (cfn);
1360 : else
1361 2633 : ifn = associated_internal_fn (fndecl);
1362 17137 : if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1363 : {
1364 13717 : const direct_internal_fn_info &info = direct_internal_fn (ifn);
1365 13717 : if (info.vectorizable)
1366 : {
1367 13717 : bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
1368 13717 : tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1369 13717 : tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1370 :
1371 : /* The type size of both the vectype_in and vectype_out should be
1372 : exactly the same when vectype_out isn't participating the optab.
1373 : While there is no restriction for type size when vectype_out
1374 : is part of the optab query. */
1375 13717 : if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
1376 17137 : return IFN_LAST;
1377 :
1378 13680 : if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1379 : OPTIMIZE_FOR_SPEED))
1380 8184 : return ifn;
1381 : }
1382 : }
1383 : return IFN_LAST;
1384 : }
1385 :
1386 :
1387 : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
1388 : gimple_stmt_iterator *);
1389 :
1390 : /* Check whether a load or store statement in the loop described by
1391 : LOOP_VINFO is possible in a loop using partial vectors. This is
1392 : testing whether the vectorizer pass has the appropriate support,
1393 : as well as whether the target does.
1394 :
1395 : VLS_TYPE says whether the statement is a load or store and VECTYPE
1396 : is the type of the vector being loaded or stored. SLP_NODE is the SLP
1397 : node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1398 : says how the load or store is going to be implemented and GROUP_SIZE
1399 : is the number of load or store statements in the containing group.
1400 : If the access is a gather load or scatter store, GS_INFO describes
1401 : its arguments. If the load or store is conditional, SCALAR_MASK is the
1402 : condition under which it occurs.
1403 :
1404 : Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1405 : vectors is not supported, otherwise record the required rgroup control
1406 : types.
1407 :
1408 : If partial vectors can be used and ELSVALS is nonzero the supported
1409 : else values will be added to the vector ELSVALS points to. */
1410 :
1411 : static void
1412 299073 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
1413 : slp_tree slp_node,
1414 : vec_load_store_type vls_type,
1415 : int group_size,
1416 : vect_load_store_data *ls,
1417 : slp_tree mask_node,
1418 : vec<int> *elsvals = nullptr)
1419 : {
1420 299073 : vect_memory_access_type memory_access_type = ls->memory_access_type;
1421 :
1422 : /* Invariant loads need no special support. */
1423 299073 : if (memory_access_type == VMAT_INVARIANT)
1424 29642 : return;
1425 :
1426 : /* Figure whether the mask is uniform. scalar_mask is used to
1427 : populate the scalar_cond_masked_set. */
1428 297881 : tree scalar_mask = NULL_TREE;
1429 297881 : if (mask_node)
1430 4906 : for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
1431 : {
1432 2484 : tree def = vect_get_slp_scalar_def (mask_node, i);
1433 2484 : if (!def
1434 2484 : || (scalar_mask && def != scalar_mask))
1435 : {
1436 : scalar_mask = NULL;
1437 : break;
1438 : }
1439 : else
1440 2463 : scalar_mask = def;
1441 : }
1442 :
1443 297881 : unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
1444 297881 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1445 297881 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
1446 297881 : machine_mode vecmode = TYPE_MODE (vectype);
1447 297881 : bool is_load = (vls_type == VLS_LOAD);
1448 297881 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
1449 : {
1450 0 : nvectors /= group_size;
1451 0 : internal_fn ifn
1452 0 : = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
1453 : elsvals)
1454 0 : : vect_store_lanes_supported (vectype, group_size, true));
1455 0 : if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
1456 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1457 0 : else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
1458 0 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1459 : scalar_mask);
1460 : else
1461 : {
1462 0 : if (dump_enabled_p ())
1463 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1464 : "can't operate on partial vectors because"
1465 : " the target doesn't have an appropriate"
1466 : " load/store-lanes instruction.\n");
1467 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1468 : }
1469 : return;
1470 : }
1471 :
1472 297881 : if (mat_gather_scatter_p (memory_access_type))
1473 : {
1474 1764 : internal_fn ifn = (is_load
1475 1764 : ? IFN_MASK_GATHER_LOAD
1476 : : IFN_MASK_SCATTER_STORE);
1477 418 : internal_fn len_ifn = (is_load
1478 : ? IFN_MASK_LEN_GATHER_LOAD
1479 : : IFN_MASK_LEN_SCATTER_STORE);
1480 1764 : stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
1481 1764 : tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
1482 1764 : ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
1483 1764 : : ls->strided_offset_vectype);
1484 1764 : tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
1485 1764 : int scale = SLP_TREE_GS_SCALE (slp_node);
1486 :
1487 : /* The following "supported" checks just verify what we established in
1488 : get_load_store_type and don't try different offset types.
1489 : Therefore, off_vectype must be a supported offset type. In case
1490 : we chose a different one use this instead. */
1491 1764 : if (ls->supported_offset_vectype)
1492 0 : off_vectype = ls->supported_offset_vectype;
1493 : /* Same for scale. */
1494 1764 : if (ls->supported_scale)
1495 0 : scale = ls->supported_scale;
1496 :
1497 1764 : if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
1498 : memory_type,
1499 : off_vectype, scale,
1500 : elsvals))
1501 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1502 1764 : else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
1503 : memory_type,
1504 : off_vectype, scale,
1505 : elsvals)
1506 1764 : || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
1507 567 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1508 : scalar_mask);
1509 : else
1510 : {
1511 1197 : if (dump_enabled_p ())
1512 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1513 : "can't operate on partial vectors because"
1514 : " the target doesn't have an appropriate"
1515 : " gather load or scatter store instruction.\n");
1516 1197 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1517 : }
1518 : return;
1519 : }
1520 :
1521 296117 : if (memory_access_type != VMAT_CONTIGUOUS)
1522 : {
1523 : /* Element X of the data must come from iteration i * VF + X of the
1524 : scalar loop. We need more work to support other mappings. */
1525 26686 : if (dump_enabled_p ())
1526 735 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1527 : "can't operate on partial vectors because an"
1528 : " access isn't contiguous.\n");
1529 26686 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1530 26686 : return;
1531 : }
1532 :
1533 269431 : if (!VECTOR_MODE_P (vecmode))
1534 : {
1535 0 : if (dump_enabled_p ())
1536 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1537 : "can't operate on partial vectors when emulating"
1538 : " vector operations.\n");
1539 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1540 0 : return;
1541 : }
1542 :
1543 : /* We might load more scalars than we need for permuting SLP loads.
1544 : We checked in get_load_store_type that the extra elements
1545 : don't leak into a new vector. */
1546 360372 : auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
1547 : {
1548 90941 : unsigned int nvectors;
1549 90941 : if (can_div_away_from_zero_p (size, nunits, &nvectors))
1550 90941 : return nvectors;
1551 : gcc_unreachable ();
1552 : };
1553 :
1554 269431 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1555 269431 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1556 269431 : machine_mode mask_mode;
1557 269431 : machine_mode vmode;
1558 269431 : bool using_partial_vectors_p = false;
1559 269431 : if (get_len_load_store_mode
1560 269431 : (vecmode, is_load, nullptr, elsvals).exists (&vmode))
1561 : {
1562 0 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1563 0 : unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
1564 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
1565 0 : using_partial_vectors_p = true;
1566 : }
1567 360372 : else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
1568 269431 : && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
1569 : elsvals))
1570 : {
1571 90941 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1572 90941 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
1573 90941 : using_partial_vectors_p = true;
1574 : }
1575 :
1576 90941 : if (!using_partial_vectors_p)
1577 : {
1578 178490 : if (dump_enabled_p ())
1579 12721 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1580 : "can't operate on partial vectors because the"
1581 : " target doesn't have the appropriate partial"
1582 : " vectorization load or store.\n");
1583 178490 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1584 : }
1585 : }
1586 :
1587 : /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1588 : form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1589 : that needs to be applied to all loads and stores in a vectorized loop.
1590 : Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1591 : otherwise return VEC_MASK & LOOP_MASK.
1592 :
1593 : MASK_TYPE is the type of both masks. If new statements are needed,
1594 : insert them before GSI. */
1595 :
1596 : tree
1597 1611 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
1598 : tree vec_mask, gimple_stmt_iterator *gsi)
1599 : {
1600 1611 : gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1601 1611 : if (!loop_mask)
1602 : return vec_mask;
1603 :
1604 141 : gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1605 :
1606 141 : if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
1607 : return vec_mask;
1608 :
1609 141 : tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1610 141 : gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1611 : vec_mask, loop_mask);
1612 :
1613 141 : gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1614 141 : return and_res;
1615 : }
1616 :
1617 : /* Determine whether we can use a gather load or scatter store to vectorize
1618 : strided load or store STMT_INFO by truncating the current offset to a
1619 : smaller width. We need to be able to construct an offset vector:
1620 :
1621 : { 0, X, X*2, X*3, ... }
1622 :
1623 : without loss of precision, where X is STMT_INFO's DR_STEP.
1624 :
1625 : Return true if this is possible, describing the gather load or scatter
1626 : store in GS_INFO. MASKED_P is true if the load or store is conditional.
1627 :
1628 : If we can use gather/scatter and ELSVALS is nonzero the supported
1629 : else values will be stored in the vector ELSVALS points to. */
1630 :
1631 : static bool
1632 64677 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
1633 : loop_vec_info loop_vinfo, bool masked_p,
1634 : gather_scatter_info *gs_info,
1635 : vec<int> *elsvals)
1636 : {
1637 64677 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1638 64677 : data_reference *dr = dr_info->dr;
1639 64677 : tree step = DR_STEP (dr);
1640 64677 : if (TREE_CODE (step) != INTEGER_CST)
1641 : {
1642 : /* ??? Perhaps we could use range information here? */
1643 29065 : if (dump_enabled_p ())
1644 205 : dump_printf_loc (MSG_NOTE, vect_location,
1645 : "cannot truncate variable step.\n");
1646 : return false;
1647 : }
1648 :
1649 : /* Get the number of bits in an element. */
1650 35612 : scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1651 35612 : unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1652 :
1653 : /* Set COUNT to the upper limit on the number of elements - 1.
1654 : Start with the maximum vectorization factor. */
1655 35612 : unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1656 :
1657 : /* Try lowering COUNT to the number of scalar latch iterations. Peeling
1658 : for alignment by masking shifts the active lanes up, and we cannot tell
1659 : yet whether peeling will be done by masking, or whether masks will be
1660 : needed at all, so be conservative and do not lower COUNT when peeling
1661 : for alignment. */
1662 35612 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1663 35612 : widest_int max_iters;
1664 35612 : if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1665 35589 : && max_loop_iterations (loop, &max_iters)
1666 70475 : && max_iters < count)
1667 2128 : count = max_iters.to_shwi ();
1668 :
1669 : /* Try scales of 1 and the element size. */
1670 35612 : unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
1671 35612 : wi::overflow_type overflow = wi::OVF_NONE;
1672 106836 : for (int i = 0; i < 2; ++i)
1673 : {
1674 71224 : unsigned int scale = scales[i];
1675 71224 : widest_int factor;
1676 71224 : if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1677 0 : continue;
1678 :
1679 : /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1680 71224 : widest_int range = wi::mul (count, factor, SIGNED, &overflow);
1681 71224 : if (overflow)
1682 0 : continue;
1683 71224 : signop sign = range >= 0 ? UNSIGNED : SIGNED;
1684 71224 : unsigned int min_offset_bits = wi::min_precision (range, sign);
1685 :
1686 : /* Find the narrowest viable offset type. */
1687 71224 : unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
1688 71224 : tree offset_type = build_nonstandard_integer_type (offset_bits,
1689 : sign == UNSIGNED);
1690 :
1691 : /* See whether the target supports the operation with an offset
1692 : no narrower than OFFSET_TYPE. */
1693 71224 : tree memory_type = TREE_TYPE (DR_REF (dr));
1694 71224 : tree tmp_offset_vectype;
1695 71224 : int tmp_scale;
1696 71224 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
1697 : vectype, memory_type, offset_type,
1698 : scale, &tmp_scale,
1699 : &gs_info->ifn, &gs_info->offset_vectype,
1700 : &tmp_offset_vectype, elsvals)
1701 71224 : || gs_info->ifn == IFN_LAST)
1702 71224 : continue;
1703 :
1704 0 : gs_info->decl = NULL_TREE;
1705 : /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1706 : but we don't need to store that here. */
1707 0 : gs_info->base = NULL_TREE;
1708 0 : gs_info->alias_ptr = build_int_cst
1709 0 : (reference_alias_ptr_type (DR_REF (dr)),
1710 0 : get_object_alignment (DR_REF (dr)));
1711 0 : gs_info->element_type = TREE_TYPE (vectype);
1712 0 : gs_info->offset = fold_convert (offset_type, step);
1713 0 : gs_info->scale = scale;
1714 0 : gs_info->memory_type = memory_type;
1715 0 : return true;
1716 71224 : }
1717 :
1718 35612 : if (overflow && dump_enabled_p ())
1719 0 : dump_printf_loc (MSG_NOTE, vect_location,
1720 : "truncating gather/scatter offset to %d bits"
1721 : " might change its value.\n", element_bits);
1722 :
1723 : return false;
1724 64677 : }
1725 :
1726 : /* Return true if we can use gather/scatter or strided internal functions
1727 : to vectorize STMT_INFO, which is a grouped or strided load or store
1728 : with multiple lanes and will be implemented by a type-punned access
1729 : of a vector with element size that matches the number of lanes.
1730 :
1731 : MASKED_P is true if load or store is conditional.
1732 : When returning true, fill in GS_INFO with the information required to
1733 : perform the operation. Also, store the punning type in PUNNED_VECTYPE.
1734 :
1735 : If successful and ELSVALS is nonzero the supported
1736 : else values will be stored in the vector ELSVALS points to. */
1737 :
1738 : static bool
1739 4658 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
1740 : loop_vec_info loop_vinfo, bool masked_p,
1741 : unsigned int nelts,
1742 : gather_scatter_info *info, vec<int> *elsvals,
1743 : tree *pun_vectype)
1744 : {
1745 4658 : data_reference *dr = dr_info->dr;
1746 :
1747 : /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
1748 : multiple gathers/scatter. */
1749 4658 : if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
1750 : return false;
1751 :
1752 : /* Pun the vectype with one of the same size but an element spanning
1753 : NELTS elements of VECTYPE.
1754 : The punned type of a V16QI with NELTS = 4 would be V4SI.
1755 : */
1756 4017 : tree tmp;
1757 4017 : unsigned int pieces;
1758 4017 : if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
1759 4017 : || pieces <= 1)
1760 2003 : return false;
1761 :
1762 2014 : *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
1763 :
1764 2014 : if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
1765 : return false;
1766 :
1767 1868 : internal_fn ifn;
1768 1868 : tree offset_vectype = *pun_vectype;
1769 :
1770 1261 : internal_fn strided_ifn = DR_IS_READ (dr)
1771 1868 : ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
1772 :
1773 : /* Check if we have a gather/scatter with the new type. We're just trying
1774 : with the type itself as offset for now. If not, check if we have a
1775 : strided load/store. These have fewer constraints (for example no offset
1776 : type must exist) so it is possible that even though a gather/scatter is
1777 : not available we still have a strided load/store. */
1778 1868 : bool ok = false;
1779 1868 : tree tmp_vectype;
1780 1868 : int tmp_scale;
1781 1868 : if (vect_gather_scatter_fn_p
1782 1868 : (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
1783 1868 : TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
1784 : &offset_vectype, &tmp_vectype, elsvals))
1785 : ok = true;
1786 1868 : else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
1787 : elsvals))
1788 : {
1789 : /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
1790 : will switch back to the strided variants. */
1791 0 : ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
1792 : IFN_MASK_LEN_SCATTER_STORE;
1793 0 : ok = true;
1794 : }
1795 :
1796 0 : if (ok)
1797 : {
1798 0 : info->ifn = ifn;
1799 0 : info->decl = NULL_TREE;
1800 0 : info->base = dr->ref;
1801 0 : info->alias_ptr = build_int_cst
1802 0 : (reference_alias_ptr_type (DR_REF (dr)),
1803 0 : get_object_alignment (DR_REF (dr)));
1804 0 : info->element_type = TREE_TYPE (*pun_vectype);
1805 0 : info->offset_vectype = offset_vectype;
1806 : /* No need to set the offset, vect_get_strided_load_store_ops
1807 : will do that. */
1808 0 : info->scale = 1;
1809 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
1810 0 : return true;
1811 : }
1812 :
1813 : return false;
1814 : }
1815 :
1816 :
1817 : /* Return true if we can use gather/scatter internal functions to
1818 : vectorize STMT_INFO, which is a grouped or strided load or store.
1819 : MASKED_P is true if load or store is conditional. When returning
1820 : true, fill in GS_INFO with the information required to perform the
1821 : operation.
1822 :
1823 : If we can use gather/scatter and ELSVALS is nonzero the supported
1824 : else values will be stored in the vector ELSVALS points to. */
1825 :
1826 : static bool
1827 64677 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
1828 : loop_vec_info loop_vinfo, bool masked_p,
1829 : gather_scatter_info *gs_info,
1830 : vec<int> *elsvals,
1831 : unsigned int group_size,
1832 : bool single_element_p)
1833 : {
1834 64677 : if (!vect_check_gather_scatter (stmt_info, vectype,
1835 : loop_vinfo, gs_info, elsvals)
1836 64677 : || gs_info->ifn == IFN_LAST)
1837 : {
1838 64677 : if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
1839 : masked_p, gs_info, elsvals))
1840 : return false;
1841 : }
1842 :
1843 0 : if (!single_element_p
1844 0 : && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
1845 : gs_info->scale,
1846 : group_size))
1847 : return false;
1848 :
1849 0 : if (dump_enabled_p ())
1850 0 : dump_printf_loc (MSG_NOTE, vect_location,
1851 : "using gather/scatter for strided/grouped access,"
1852 : " scale = %d\n", gs_info->scale);
1853 :
1854 : return true;
1855 : }
1856 :
1857 : /* STMT_INFO is a non-strided load or store, meaning that it accesses
1858 : elements with a known constant step. Return -1 if that step
1859 : is negative, 0 if it is zero, and 1 if it is greater than zero. */
1860 :
1861 : int
1862 1527963 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
1863 : {
1864 1527963 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1865 1527963 : return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
1866 1527963 : size_zero_node);
1867 : }
1868 :
1869 : /* If the target supports a permute mask that reverses the elements in
1870 : a vector of type VECTYPE, return that mask, otherwise return null. */
1871 :
1872 : tree
1873 9281 : perm_mask_for_reverse (tree vectype)
1874 : {
1875 9281 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1876 :
1877 : /* The encoding has a single stepped pattern. */
1878 9281 : vec_perm_builder sel (nunits, 1, 3);
1879 46405 : for (int i = 0; i < 3; ++i)
1880 27843 : sel.quick_push (nunits - 1 - i);
1881 :
1882 9281 : vec_perm_indices indices (sel, 1, nunits);
1883 9281 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
1884 : indices))
1885 : return NULL_TREE;
1886 8101 : return vect_gen_perm_mask_checked (vectype, indices);
1887 9281 : }
1888 :
1889 : /* A subroutine of get_load_store_type, with a subset of the same
1890 : arguments. Handle the case where STMT_INFO is a load or store that
1891 : accesses consecutive elements with a negative step. Sets *POFFSET
1892 : to the offset to be applied to the DR for the first access. */
1893 :
1894 : static vect_memory_access_type
1895 12369 : get_negative_load_store_type (vec_info *vinfo,
1896 : stmt_vec_info stmt_info, tree vectype,
1897 : vec_load_store_type vls_type,
1898 : unsigned int ncopies, poly_int64 *poffset)
1899 : {
1900 12369 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1901 12369 : dr_alignment_support alignment_support_scheme;
1902 :
1903 12369 : if (ncopies > 1)
1904 : {
1905 0 : if (dump_enabled_p ())
1906 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1907 : "multiple types with negative step.\n");
1908 : return VMAT_ELEMENTWISE;
1909 : }
1910 :
1911 : /* For backward running DRs the first access in vectype actually is
1912 : N-1 elements before the address of the DR. */
1913 12369 : *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
1914 12369 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1915 :
1916 12369 : int misalignment = dr_misalignment (dr_info, vectype, *poffset);
1917 12369 : alignment_support_scheme
1918 12369 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
1919 12369 : if (alignment_support_scheme != dr_aligned
1920 12369 : && alignment_support_scheme != dr_unaligned_supported)
1921 : {
1922 4500 : if (dump_enabled_p ())
1923 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1924 : "negative step but alignment required.\n");
1925 4500 : *poffset = 0;
1926 4500 : return VMAT_ELEMENTWISE;
1927 : }
1928 :
1929 7869 : if (vls_type == VLS_STORE_INVARIANT)
1930 : {
1931 1181 : if (dump_enabled_p ())
1932 21 : dump_printf_loc (MSG_NOTE, vect_location,
1933 : "negative step with invariant source;"
1934 : " no permute needed.\n");
1935 : return VMAT_CONTIGUOUS_DOWN;
1936 : }
1937 :
1938 6688 : if (!perm_mask_for_reverse (vectype))
1939 : {
1940 1180 : if (dump_enabled_p ())
1941 52 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1942 : "negative step and reversing not supported.\n");
1943 1180 : *poffset = 0;
1944 1180 : return VMAT_ELEMENTWISE;
1945 : }
1946 :
1947 : return VMAT_CONTIGUOUS_REVERSE;
1948 : }
1949 :
1950 : /* STMT_INFO is either a masked or unconditional store. Return the value
1951 : being stored. */
1952 :
1953 : tree
1954 0 : vect_get_store_rhs (stmt_vec_info stmt_info)
1955 : {
1956 0 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
1957 : {
1958 0 : gcc_assert (gimple_assign_single_p (assign));
1959 0 : return gimple_assign_rhs1 (assign);
1960 : }
1961 0 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
1962 : {
1963 0 : internal_fn ifn = gimple_call_internal_fn (call);
1964 0 : int index = internal_fn_stored_value_index (ifn);
1965 0 : gcc_assert (index >= 0);
1966 0 : return gimple_call_arg (call, index);
1967 : }
1968 0 : gcc_unreachable ();
1969 : }
1970 :
1971 : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1972 :
1973 : This function returns a vector type which can be composed with NELTS pieces,
1974 : whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1975 : same vector size as the return vector. It checks target whether supports
1976 : pieces-size vector mode for construction firstly, if target fails to, check
1977 : pieces-size scalar mode for construction further. It returns NULL_TREE if
1978 : fails to find the available composition. If the caller only wants scalar
1979 : pieces where PTYPE e.g. is a possible gather/scatter element type
1980 : SCALAR_PTYPE_ONLY must be true.
1981 :
1982 : For example, for (vtype=V16QI, nelts=4), we can probably get:
1983 : - V16QI with PTYPE V4QI.
1984 : - V4SI with PTYPE SI.
1985 : - NULL_TREE. */
1986 :
1987 : static tree
1988 12838 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
1989 : bool scalar_ptype_only)
1990 : {
1991 12838 : gcc_assert (VECTOR_TYPE_P (vtype));
1992 12838 : gcc_assert (known_gt (nelts, 0U));
1993 :
1994 12838 : machine_mode vmode = TYPE_MODE (vtype);
1995 12838 : if (!VECTOR_MODE_P (vmode))
1996 : return NULL_TREE;
1997 :
1998 : /* When we are asked to compose the vector from its components let
1999 : that happen directly. */
2000 12838 : if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
2001 : {
2002 6156 : *ptype = TREE_TYPE (vtype);
2003 6156 : return vtype;
2004 : }
2005 :
2006 13364 : poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
2007 6682 : unsigned int pbsize;
2008 6682 : if (constant_multiple_p (vbsize, nelts, &pbsize))
2009 : {
2010 : /* First check if vec_init optab supports construction from
2011 : vector pieces directly. */
2012 6682 : scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
2013 13364 : poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
2014 6682 : machine_mode rmode;
2015 6682 : if (!scalar_ptype_only
2016 8228 : && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
2017 10888 : && (convert_optab_handler (vec_init_optab, vmode, rmode)
2018 : != CODE_FOR_nothing))
2019 : {
2020 3560 : *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
2021 3560 : return vtype;
2022 : }
2023 :
2024 : /* Otherwise check if exists an integer type of the same piece size and
2025 : if vec_init optab supports construction from it directly. */
2026 3122 : if (int_mode_for_size (pbsize, 0).exists (&elmode)
2027 3122 : && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
2028 : {
2029 2938 : if (scalar_ptype_only
2030 2938 : || convert_optab_handler (vec_init_optab, rmode, elmode)
2031 : != CODE_FOR_nothing)
2032 : {
2033 2938 : *ptype = build_nonstandard_integer_type (pbsize, 1);
2034 2938 : return build_vector_type (*ptype, nelts);
2035 : }
2036 : }
2037 : }
2038 :
2039 : return NULL_TREE;
2040 : }
2041 :
2042 : /* Check if the load permutation of NODE only refers to a consecutive
2043 : subset of the group indices. We also require the length of the
2044 : permutation to be a power of two.
2045 : Such load permutations can be elided in strided access schemes as
2046 : we can "jump over" the gap they leave. */
2047 :
2048 : static bool
2049 45633 : has_consecutive_load_permutation (slp_tree node)
2050 : {
2051 45633 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
2052 45633 : if (!perm.exists ()
2053 2183 : || perm.length () <= 1
2054 46142 : || !pow2p_hwi (perm.length ()))
2055 : return false;
2056 :
2057 489 : return vect_load_perm_consecutive_p (node);
2058 : }
2059 :
2060 :
2061 : /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
2062 : if there is a memory access type that the vectorized form can use,
2063 : storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2064 : or scatters, fill in GS_INFO accordingly. In addition
2065 : *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2066 : the target does not support the alignment scheme. *MISALIGNMENT
2067 : is set according to the alignment of the access (including
2068 : DR_MISALIGNMENT_UNKNOWN when it is unknown).
2069 :
2070 : MASKED_P is true if the statement is conditional on a vectorized mask.
2071 : VECTYPE is the vector type that the vectorized statements will use.
2072 :
2073 : If ELSVALS is nonzero the supported else values will be stored in the
2074 : vector ELSVALS points to. */
2075 :
2076 : static bool
2077 1413179 : get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
2078 : tree vectype, slp_tree slp_node,
2079 : bool masked_p, vec_load_store_type vls_type,
2080 : vect_load_store_data *ls)
2081 : {
2082 1413179 : vect_memory_access_type *memory_access_type = &ls->memory_access_type;
2083 1413179 : poly_int64 *poffset = &ls->poffset;
2084 1413179 : dr_alignment_support *alignment_support_scheme
2085 : = &ls->alignment_support_scheme;
2086 1413179 : int *misalignment = &ls->misalignment;
2087 1413179 : internal_fn *lanes_ifn = &ls->lanes_ifn;
2088 1413179 : vec<int> *elsvals = &ls->elsvals;
2089 1413179 : tree *ls_type = &ls->ls_type;
2090 1413179 : bool *slp_perm = &ls->slp_perm;
2091 1413179 : unsigned *n_perms = &ls->n_perms;
2092 1413179 : unsigned *n_loads = &ls->n_loads;
2093 1413179 : tree *supported_offset_vectype = &ls->supported_offset_vectype;
2094 1413179 : int *supported_scale = &ls->supported_scale;
2095 1413179 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
2096 1413179 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2097 1413179 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2098 1413179 : stmt_vec_info first_stmt_info;
2099 1413179 : unsigned int group_size;
2100 1413179 : unsigned HOST_WIDE_INT gap;
2101 1413179 : bool single_element_p;
2102 1413179 : poly_int64 neg_ldst_offset = 0;
2103 :
2104 1413179 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2105 1413179 : *poffset = 0;
2106 1413179 : *ls_type = NULL_TREE;
2107 1413179 : *slp_perm = false;
2108 1413179 : *n_perms = -1U;
2109 1413179 : *n_loads = -1U;
2110 1413179 : ls->subchain_p = false;
2111 :
2112 1413179 : bool perm_ok = true;
2113 1413179 : poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
2114 :
2115 1413179 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2116 81024 : perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
2117 81024 : vf, true, n_perms, n_loads);
2118 :
2119 1413179 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2120 : {
2121 902867 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
2122 902867 : group_size = DR_GROUP_SIZE (first_stmt_info);
2123 902867 : gap = DR_GROUP_GAP (first_stmt_info);
2124 902867 : single_element_p = (stmt_info == first_stmt_info
2125 902867 : && !DR_GROUP_NEXT_ELEMENT (stmt_info));
2126 : }
2127 : else
2128 : {
2129 : first_stmt_info = stmt_info;
2130 : group_size = 1;
2131 : gap = 0;
2132 : single_element_p = true;
2133 : }
2134 1413179 : dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
2135 :
2136 : /* There can only be a gap at the end of the group if the stride is
2137 : known at compile time. */
2138 1413179 : gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
2139 :
2140 : /* For SLP vectorization we directly vectorize a subchain
2141 : without permutation. */
2142 1413179 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2143 1332155 : first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
2144 :
2145 1413179 : if (STMT_VINFO_STRIDED_P (first_stmt_info))
2146 : {
2147 : /* Try to use consecutive accesses of as many elements as possible,
2148 : separated by the stride, until we have a complete vector.
2149 : Fall back to scalar accesses if that isn't possible. */
2150 45633 : *memory_access_type = VMAT_STRIDED_SLP;
2151 :
2152 : /* If the load permutation is consecutive we can reduce the group to
2153 : the elements the permutation accesses. Then we release the
2154 : permutation. */
2155 45633 : if (has_consecutive_load_permutation (slp_node))
2156 : {
2157 51 : ls->subchain_p = true;
2158 51 : group_size = SLP_TREE_LANES (slp_node);
2159 51 : SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
2160 : }
2161 : }
2162 1367546 : else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2163 : {
2164 11212 : slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
2165 11212 : tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
2166 11212 : int scale = SLP_TREE_GS_SCALE (slp_node);
2167 11212 : tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
2168 11212 : tree tem;
2169 11212 : if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
2170 : masked_p, vectype, memory_type,
2171 : offset_vectype, scale, supported_scale,
2172 : &ls->gs.ifn, &tem,
2173 : supported_offset_vectype, elsvals))
2174 : {
2175 0 : if (dump_enabled_p ())
2176 : {
2177 0 : dump_printf_loc (MSG_NOTE, vect_location,
2178 : "gather/scatter with required "
2179 : "offset type "
2180 : "%T and offset scale %d.\n",
2181 : offset_vectype, scale);
2182 0 : if (*supported_offset_vectype)
2183 0 : dump_printf_loc (MSG_NOTE, vect_location,
2184 : " target supports offset type %T.\n",
2185 : *supported_offset_vectype);
2186 0 : if (*supported_scale)
2187 0 : dump_printf_loc (MSG_NOTE, vect_location,
2188 : " target supports offset scale %d.\n",
2189 : *supported_scale);
2190 : }
2191 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2192 : }
2193 11212 : else if (vls_type == VLS_LOAD
2194 11212 : ? (targetm.vectorize.builtin_gather
2195 9621 : && (ls->gs.decl
2196 9621 : = targetm.vectorize.builtin_gather (vectype,
2197 9621 : TREE_TYPE
2198 : (offset_vectype),
2199 : scale)))
2200 1591 : : (targetm.vectorize.builtin_scatter
2201 1591 : && (ls->gs.decl
2202 1591 : = targetm.vectorize.builtin_scatter (vectype,
2203 1591 : TREE_TYPE
2204 : (offset_vectype),
2205 : scale))))
2206 575 : *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
2207 : else
2208 : {
2209 : /* GATHER_SCATTER_EMULATED_P. */
2210 10637 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
2211 10637 : || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
2212 10637 : || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
2213 10637 : || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
2214 13857 : TYPE_VECTOR_SUBPARTS (vectype)))
2215 : {
2216 3220 : if (dump_enabled_p ())
2217 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2218 : "unsupported vector types for emulated "
2219 : "gather.\n");
2220 3220 : return false;
2221 : }
2222 7417 : *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
2223 : }
2224 : }
2225 : else
2226 : {
2227 1356334 : int cmp = compare_step_with_zero (vinfo, stmt_info);
2228 1356334 : if (cmp < 0)
2229 : {
2230 12553 : if (single_element_p)
2231 : /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2232 : only correct for single element "interleaving" SLP. */
2233 12369 : *memory_access_type = get_negative_load_store_type
2234 12369 : (vinfo, stmt_info, vectype, vls_type, 1,
2235 : &neg_ldst_offset);
2236 : else
2237 : /* We can fall back to VMAT_STRIDED_SLP since that does
2238 : not care whether the stride between the group instances
2239 : is positive or negative. */
2240 184 : *memory_access_type = VMAT_STRIDED_SLP;
2241 : }
2242 1343781 : else if (cmp == 0 && loop_vinfo)
2243 : {
2244 3407 : gcc_assert (vls_type == VLS_LOAD);
2245 3407 : *memory_access_type = VMAT_INVARIANT;
2246 : }
2247 : /* Try using LOAD/STORE_LANES. */
2248 1340374 : else if (slp_node->ldst_lanes
2249 1340374 : && (*lanes_ifn
2250 0 : = (vls_type == VLS_LOAD
2251 0 : ? vect_load_lanes_supported (vectype, group_size,
2252 : masked_p, elsvals)
2253 0 : : vect_store_lanes_supported (vectype, group_size,
2254 : masked_p))) != IFN_LAST)
2255 0 : *memory_access_type = VMAT_LOAD_STORE_LANES;
2256 1340374 : else if (!loop_vinfo && slp_node->avoid_stlf_fail)
2257 : {
2258 70 : *memory_access_type = VMAT_ELEMENTWISE;
2259 70 : if (dump_enabled_p ())
2260 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2261 : "using element-wise load to avoid disrupting "
2262 : "cross iteration store-to-load forwarding\n");
2263 : }
2264 : else
2265 1340304 : *memory_access_type = VMAT_CONTIGUOUS;
2266 :
2267 : /* True if the vectorized statements would access beyond the last
2268 : statement in the group. */
2269 1356334 : bool overrun_p = false;
2270 :
2271 : /* True if we can cope with such overrun by peeling for gaps, so that
2272 : there is at least one final scalar iteration after the vector loop. */
2273 2712668 : bool can_overrun_p = (!masked_p
2274 1356334 : && vls_type == VLS_LOAD
2275 547178 : && loop_vinfo
2276 403944 : && !loop->inner
2277 1758444 : && (*memory_access_type != VMAT_STRIDED_SLP
2278 80 : || cmp > 0));
2279 :
2280 : /* If this is single-element interleaving with an element
2281 : distance that leaves unused vector loads around fall back
2282 : to elementwise access if possible - we otherwise least
2283 : create very sub-optimal code in that case (and
2284 : blow up memory, see PR65518). */
2285 1356334 : if (loop_vinfo
2286 1356334 : && single_element_p
2287 490365 : && (*memory_access_type == VMAT_CONTIGUOUS
2288 15776 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2289 1846699 : && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
2290 : {
2291 17997 : *memory_access_type = VMAT_ELEMENTWISE;
2292 17997 : if (dump_enabled_p ())
2293 198 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2294 : "single-element interleaving not supported "
2295 : "for not adjacent vector loads, using "
2296 : "elementwise access\n");
2297 : }
2298 :
2299 : /* Also fall back to elementwise access in case we did not lower a
2300 : permutation and cannot code generate it. */
2301 1356334 : if (loop_vinfo
2302 545042 : && *memory_access_type != VMAT_ELEMENTWISE
2303 521365 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2304 1385128 : && !perm_ok)
2305 : {
2306 2090 : *memory_access_type = VMAT_ELEMENTWISE;
2307 2090 : if (dump_enabled_p ())
2308 248 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2309 : "permutation not supported, using elementwise "
2310 : "access\n");
2311 : }
2312 :
2313 545042 : overrun_p = (loop_vinfo && gap != 0
2314 1399679 : && *memory_access_type != VMAT_ELEMENTWISE);
2315 1356334 : if (overrun_p && vls_type != VLS_LOAD)
2316 : {
2317 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2318 : "Grouped store with gaps requires"
2319 : " non-consecutive accesses\n");
2320 0 : return false;
2321 : }
2322 :
2323 1356334 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
2324 1356334 : poly_int64 off = 0;
2325 1356334 : if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2326 5349 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2327 :
2328 : /* An overrun is fine if the trailing elements are smaller
2329 : than the alignment boundary B. Every vector access will
2330 : be a multiple of B and so we are guaranteed to access a
2331 : non-gap element in the same B-sized block. */
2332 1356334 : if (overrun_p
2333 1356334 : && gap < (vect_known_alignment_in_bytes (first_dr_info,
2334 23189 : vectype, off) / dr_size))
2335 : overrun_p = false;
2336 :
2337 : /* When we have a contiguous access across loop iterations
2338 : but the access in the loop doesn't cover the full vector
2339 : we can end up with no gap recorded but still excess
2340 : elements accessed, see PR103116. Make sure we peel for
2341 : gaps if necessary and sufficient and give up if not.
2342 :
2343 : If there is a combination of the access not covering the full
2344 : vector and a gap recorded then we may need to peel twice. */
2345 1356334 : bool large_vector_overrun_p = false;
2346 1356334 : if (loop_vinfo
2347 545042 : && (*memory_access_type == VMAT_CONTIGUOUS
2348 35883 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2349 514508 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2350 1382654 : && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
2351 : nunits))
2352 : large_vector_overrun_p = overrun_p = true;
2353 :
2354 : /* If the gap splits the vector in half and the target
2355 : can do half-vector operations avoid the epilogue peeling
2356 : by simply loading half of the vector only. Usually
2357 : the construction with an upper zero half will be elided. */
2358 1356334 : dr_alignment_support alss;
2359 1356334 : int misalign = dr_misalignment (first_dr_info, vectype, off);
2360 1356334 : tree half_vtype;
2361 1356334 : poly_uint64 remain;
2362 1356334 : unsigned HOST_WIDE_INT tem, num;
2363 1356334 : if (overrun_p
2364 1356334 : && !masked_p
2365 17678 : && (*memory_access_type == VMAT_CONTIGUOUS
2366 356 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2367 17664 : && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
2368 : vectype, misalign)))
2369 : == dr_aligned
2370 15128 : || alss == dr_unaligned_supported)
2371 9998 : && can_div_trunc_p (group_size
2372 9998 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2373 : nunits, &tem, &remain)
2374 1366332 : && (known_eq (remain, 0u)
2375 7493 : || (known_ne (remain, 0u)
2376 5793 : && constant_multiple_p (nunits, remain, &num)
2377 1356334 : && (vector_vector_composition_type (vectype, num, &half_vtype)
2378 : != NULL_TREE))))
2379 : overrun_p = false;
2380 :
2381 1356334 : if (overrun_p && !can_overrun_p)
2382 : {
2383 20 : if (dump_enabled_p ())
2384 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2385 : "Peeling for outer loop is not supported\n");
2386 : return false;
2387 : }
2388 :
2389 : /* Peeling for gaps assumes that a single scalar iteration
2390 : is enough to make sure the last vector iteration doesn't
2391 : access excess elements. */
2392 1356314 : if (overrun_p
2393 1356314 : && (!can_div_trunc_p (group_size
2394 9360 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2395 : nunits, &tem, &remain)
2396 9360 : || maybe_lt (remain + group_size, nunits)))
2397 : {
2398 : /* But peeling a single scalar iteration is enough if
2399 : we can use the next power-of-two sized partial
2400 : access and that is sufficiently small to be covered
2401 : by the single scalar iteration. */
2402 16 : unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
2403 16 : if (masked_p
2404 16 : || !nunits.is_constant (&cnunits)
2405 16 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
2406 16 : || (((cremain = (group_size * cvf - gap) % cnunits), true)
2407 16 : && ((cpart_size = (1 << ceil_log2 (cremain))), true)
2408 16 : && (cremain + group_size < cpart_size
2409 16 : || (vector_vector_composition_type (vectype,
2410 13 : cnunits / cpart_size,
2411 : &half_vtype)
2412 : == NULL_TREE))))
2413 : {
2414 : /* If all fails we can still resort to niter masking unless
2415 : the vectors used are too big, so enforce the use of
2416 : partial vectors. */
2417 3 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2418 3 : && !large_vector_overrun_p)
2419 : {
2420 0 : if (dump_enabled_p ())
2421 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2422 : "peeling for gaps insufficient for "
2423 : "access unless using partial "
2424 : "vectors\n");
2425 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2426 : }
2427 : else
2428 : {
2429 3 : if (dump_enabled_p ())
2430 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2431 : "peeling for gaps insufficient for "
2432 : "access\n");
2433 23 : return false;
2434 : }
2435 : }
2436 13 : else if (large_vector_overrun_p)
2437 : {
2438 13 : if (dump_enabled_p ())
2439 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2440 : "can't operate on partial vectors because "
2441 : "only unmasked loads handle access "
2442 : "shortening required because of gaps at "
2443 : "the end of the access\n");
2444 13 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2445 : }
2446 : }
2447 :
2448 1356311 : if (overrun_p)
2449 : {
2450 9357 : gcc_assert (can_overrun_p);
2451 9357 : if (dump_enabled_p ())
2452 511 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2453 : "Data access with gaps requires scalar "
2454 : "epilogue loop\n");
2455 9357 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2456 : }
2457 : }
2458 :
2459 : /* As a last resort, trying using a gather load or scatter store.
2460 :
2461 : ??? Although the code can handle all group sizes correctly,
2462 : it probably isn't a win to use separate strided accesses based
2463 : on nearby locations. Or, even if it's a win over scalar code,
2464 : it might not be a win over vectorizing at a lower VF, if that
2465 : allows us to use contiguous accesses. */
2466 1409936 : vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
2467 1409936 : if (loop_vinfo
2468 598644 : && (*memory_access_type == VMAT_ELEMENTWISE
2469 598644 : || *memory_access_type == VMAT_STRIDED_SLP))
2470 : {
2471 71565 : gather_scatter_info gs_info;
2472 71565 : tree tem;
2473 71565 : if (SLP_TREE_LANES (slp_node) == 1
2474 66713 : && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2475 21795 : || single_element_p)
2476 136242 : && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
2477 : masked_p, &gs_info, elsvals,
2478 : group_size, single_element_p))
2479 : {
2480 : /* vect_use_strided_gather_scatters_p does not save the actually
2481 : supported scale and offset type so do that here.
2482 : We need it later in check_load_store_for_partial_vectors
2483 : where we only check if the given internal function is supported
2484 : (to choose whether to use the IFN, LEGACY, or EMULATED flavor
2485 : of gather/scatter) and don't re-do the full analysis. */
2486 0 : tree tmp;
2487 0 : gcc_assert (vect_gather_scatter_fn_p
2488 : (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
2489 : gs_info.memory_type, TREE_TYPE (gs_info.offset),
2490 : gs_info.scale, supported_scale, &gs_info.ifn,
2491 : &tmp, supported_offset_vectype, elsvals));
2492 :
2493 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2494 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2495 0 : ls->gs.ifn = gs_info.ifn;
2496 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2497 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2498 : }
2499 71565 : else if (SLP_TREE_LANES (slp_node) > 1
2500 : && !masked_p
2501 4852 : && !single_element_p
2502 76223 : && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
2503 : vectype, loop_vinfo,
2504 : masked_p, group_size,
2505 : &gs_info, elsvals, &tem))
2506 : {
2507 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2508 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2509 0 : grouped_gather_fallback = *memory_access_type;
2510 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2511 0 : ls->gs.ifn = gs_info.ifn;
2512 0 : vectype = *ls_type = tem;
2513 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2514 : }
2515 : }
2516 :
2517 1409936 : if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
2518 1409936 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2519 6526 : *poffset = neg_ldst_offset;
2520 :
2521 1409936 : if (*memory_access_type == VMAT_ELEMENTWISE
2522 1384099 : || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
2523 1383524 : || *memory_access_type == VMAT_STRIDED_SLP
2524 1337726 : || *memory_access_type == VMAT_INVARIANT)
2525 : {
2526 75617 : *alignment_support_scheme = dr_unaligned_supported;
2527 75617 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2528 : }
2529 : else
2530 : {
2531 1334319 : if (mat_gather_scatter_p (*memory_access_type)
2532 : && !first_dr_info)
2533 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2534 : else
2535 1334319 : *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
2536 1334319 : *alignment_support_scheme
2537 1334319 : = vect_supportable_dr_alignment
2538 1334319 : (vinfo, first_dr_info, vectype, *misalignment,
2539 : mat_gather_scatter_p (*memory_access_type));
2540 1334319 : if (grouped_gather_fallback != VMAT_UNINITIALIZED
2541 0 : && *alignment_support_scheme != dr_aligned
2542 0 : && *alignment_support_scheme != dr_unaligned_supported)
2543 : {
2544 : /* No supportable alignment for a grouped gather, fall back to the
2545 : original memory access type. Even though VMAT_STRIDED_SLP might
2546 : also try aligned vector loads it can still choose vector
2547 : construction from scalars. */
2548 0 : *memory_access_type = grouped_gather_fallback;
2549 0 : *alignment_support_scheme = dr_unaligned_supported;
2550 0 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2551 : }
2552 : }
2553 :
2554 1409936 : if ((*memory_access_type == VMAT_ELEMENTWISE
2555 1409936 : || *memory_access_type == VMAT_STRIDED_SLP)
2556 : && !nunits.is_constant ())
2557 : {
2558 : if (dump_enabled_p ())
2559 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2560 : "Not using elementwise accesses due to variable "
2561 : "vectorization factor.\n");
2562 : return false;
2563 : }
2564 :
2565 : /* Checks if all scalar iterations are known to be inbounds. */
2566 1409936 : bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
2567 :
2568 : /* Check if we support the operation if early breaks are needed. Here we
2569 : must ensure that we don't access any more than the scalar code would
2570 : have. A masked operation would ensure this, so for these load types
2571 : force masking. */
2572 1409936 : if (loop_vinfo
2573 598644 : && dr_safe_speculative_read_required (stmt_info)
2574 1605358 : && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2575 : {
2576 195422 : if (mat_gather_scatter_p (*memory_access_type)
2577 195422 : || *memory_access_type == VMAT_STRIDED_SLP)
2578 : {
2579 9472 : if (dump_enabled_p ())
2580 8 : dump_printf_loc (MSG_NOTE, vect_location,
2581 : "early break not supported: cannot peel for "
2582 : "alignment. With non-contiguous memory vectorization"
2583 : " could read out of bounds at %G ",
2584 : STMT_VINFO_STMT (stmt_info));
2585 9472 : if (inbounds)
2586 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2587 : else
2588 : return false;
2589 : }
2590 : /* Block-level alignment: Even though individual accesses of
2591 : VMAT_ELEMENTWISE type do not cause alignment problems, loading the
2592 : whole vector's worth of values in a speculative early-break context
2593 : might cross a page boundary. Set the alignment scheme to `dr_aligned'
2594 : here in order to force checking of whether such accesses meet
2595 : alignment criteria. */
2596 185950 : else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
2597 15095 : *alignment_support_scheme = dr_aligned;
2598 : }
2599 :
2600 : /* If this DR needs alignment for correctness, we must ensure the target
2601 : alignment is a constant power-of-two multiple of the amount read per
2602 : vector iteration or force masking. */
2603 1400464 : if (dr_safe_speculative_read_required (stmt_info)
2604 1400464 : && (*alignment_support_scheme == dr_aligned
2605 111329 : && !mat_gather_scatter_p (*memory_access_type)))
2606 : {
2607 : /* We can only peel for loops, of course. */
2608 111329 : gcc_checking_assert (loop_vinfo);
2609 :
2610 111329 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2611 111329 : poly_uint64 read_amount
2612 111329 : = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
2613 111329 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2614 111329 : read_amount *= group_size;
2615 :
2616 111329 : auto target_alignment
2617 111329 : = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
2618 111329 : if (!multiple_p (target_alignment, read_amount))
2619 : {
2620 12788 : if (dump_enabled_p ())
2621 : {
2622 28 : dump_printf_loc (MSG_NOTE, vect_location,
2623 : "desired alignment not met, target was ");
2624 28 : dump_dec (MSG_NOTE, target_alignment);
2625 28 : dump_printf (MSG_NOTE, " previously, but read amount is ");
2626 28 : dump_dec (MSG_NOTE, read_amount);
2627 28 : dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
2628 : }
2629 15054 : return false;
2630 : }
2631 :
2632 : /* When using a group access the first element may be aligned but the
2633 : subsequent loads may not be. For LOAD_LANES since the loads are based
2634 : on the first DR then all loads in the group are aligned. For
2635 : non-LOAD_LANES this is not the case. In particular a load + blend when
2636 : there are gaps can have the non first loads issued unaligned, even
2637 : partially overlapping the memory of the first load in order to simplify
2638 : the blend. This is what the x86_64 backend does for instance. As
2639 : such only the first load in the group is aligned, the rest are not.
2640 : Because of this the permutes may break the alignment requirements that
2641 : have been set, and as such we should for now, reject them. */
2642 98541 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2643 : {
2644 2266 : if (dump_enabled_p ())
2645 87 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2646 : "loads with load permutations not supported for "
2647 : "speculative early break loads for %G",
2648 : STMT_VINFO_STMT (stmt_info));
2649 : return false;
2650 : }
2651 :
2652 : /* Reject vectorization if we know the read mount per vector iteration
2653 : exceeds the min page size. */
2654 96275 : if (known_gt (read_amount, (unsigned) param_min_pagesize))
2655 : {
2656 0 : if (dump_enabled_p ())
2657 : {
2658 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2659 : "alignment required for correctness (");
2660 0 : dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
2661 0 : dump_printf (MSG_NOTE, ") may exceed page size.\n");
2662 : }
2663 : return false;
2664 : }
2665 :
2666 96275 : if (!vf.is_constant ())
2667 : {
2668 : /* For VLA modes, we need a runtime check to ensure any speculative
2669 : read amount does not exceed the page size. Here we record the max
2670 : possible read amount for the check. */
2671 : if (maybe_gt (read_amount,
2672 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
2673 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
2674 :
2675 : /* For VLA modes, we must use partial vectors. */
2676 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2677 : }
2678 : }
2679 :
2680 1385410 : if (*alignment_support_scheme == dr_unaligned_unsupported)
2681 : {
2682 71113 : if (dump_enabled_p ())
2683 256 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2684 : "unsupported unaligned access\n");
2685 : return false;
2686 : }
2687 :
2688 : /* FIXME: At the moment the cost model seems to underestimate the
2689 : cost of using elementwise accesses. This check preserves the
2690 : traditional behavior until that can be fixed. */
2691 1314297 : if (*memory_access_type == VMAT_ELEMENTWISE
2692 15023 : && !STMT_VINFO_STRIDED_P (first_stmt_info)
2693 1329320 : && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
2694 9766 : && single_element_p
2695 9119 : && !pow2p_hwi (group_size)))
2696 : {
2697 9310 : if (dump_enabled_p ())
2698 364 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2699 : "not falling back to elementwise accesses\n");
2700 : return false;
2701 : }
2702 :
2703 : /* For BB vectorization build up the vector from existing scalar defs. */
2704 1304987 : if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
2705 : return false;
2706 :
2707 : /* Some loads need to explicitly permute the loaded data if there
2708 : is a load permutation. Among those are:
2709 : - VMAT_ELEMENTWISE.
2710 : - VMAT_STRIDED_SLP.
2711 : - VMAT_GATHER_SCATTER:
2712 : - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
2713 : - Grouped strided gather (ditto but for #lanes > 1).
2714 :
2715 : For VMAT_ELEMENTWISE we can fold the load permutation into the
2716 : individual indices we access directly, eliding the permutation.
2717 : Strided gather only allows load permutations for the
2718 : single-element case. */
2719 :
2720 1304987 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2721 1304987 : && !(*memory_access_type == VMAT_ELEMENTWISE
2722 53739 : || (mat_gather_scatter_p (*memory_access_type)
2723 0 : && SLP_TREE_LANES (slp_node) == 1
2724 0 : && single_element_p)))
2725 : {
2726 53739 : if (!loop_vinfo)
2727 : {
2728 : /* In BB vectorization we may not actually use a loaded vector
2729 : accessing elements in excess of DR_GROUP_SIZE. */
2730 31743 : stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
2731 31743 : group_info = DR_GROUP_FIRST_ELEMENT (group_info);
2732 31743 : unsigned HOST_WIDE_INT nunits;
2733 31743 : unsigned j, k, maxk = 0;
2734 111127 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
2735 79384 : if (k > maxk)
2736 : maxk = k;
2737 31743 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2738 57712 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
2739 31743 : || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
2740 : {
2741 5774 : if (dump_enabled_p ())
2742 41 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2743 : "BB vectorization with gaps at the end of "
2744 : "a load is not supported\n");
2745 1413179 : return false;
2746 : }
2747 : }
2748 :
2749 47965 : if (!perm_ok)
2750 : {
2751 2343 : if (dump_enabled_p ())
2752 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
2753 : vect_location,
2754 : "unsupported load permutation\n");
2755 : return false;
2756 : }
2757 :
2758 45622 : *slp_perm = true;
2759 : }
2760 :
2761 : return true;
2762 : }
2763 :
2764 : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2765 : conditional operation STMT_INFO. When returning true, store the mask
2766 : in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
2767 : the vectorized mask in *MASK_VECTYPE_OUT. */
2768 :
2769 : static bool
2770 12339 : vect_check_scalar_mask (vec_info *vinfo,
2771 : slp_tree slp_node, unsigned mask_index,
2772 : slp_tree *mask_node,
2773 : vect_def_type *mask_dt_out, tree *mask_vectype_out)
2774 : {
2775 12339 : enum vect_def_type mask_dt;
2776 12339 : tree mask_vectype;
2777 12339 : slp_tree mask_node_1;
2778 12339 : tree mask_;
2779 12339 : if (!vect_is_simple_use (vinfo, slp_node, mask_index,
2780 : &mask_, &mask_node_1, &mask_dt, &mask_vectype))
2781 : {
2782 0 : if (dump_enabled_p ())
2783 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2784 : "mask use not simple.\n");
2785 : return false;
2786 : }
2787 :
2788 12339 : if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
2789 12339 : && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
2790 : {
2791 0 : if (dump_enabled_p ())
2792 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2793 : "mask argument is not a boolean.\n");
2794 : return false;
2795 : }
2796 :
2797 12339 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2798 12339 : if (!mask_vectype)
2799 19 : mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
2800 : mask_node_1);
2801 :
2802 12339 : if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2803 : {
2804 0 : if (dump_enabled_p ())
2805 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2806 : "could not find an appropriate vector mask type.\n");
2807 : return false;
2808 : }
2809 :
2810 12339 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2811 24678 : TYPE_VECTOR_SUBPARTS (vectype)))
2812 : {
2813 0 : if (dump_enabled_p ())
2814 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2815 : "vector mask type %T"
2816 : " does not match vector data type %T.\n",
2817 : mask_vectype, vectype);
2818 :
2819 : return false;
2820 : }
2821 :
2822 12339 : *mask_dt_out = mask_dt;
2823 12339 : *mask_vectype_out = mask_vectype;
2824 12339 : *mask_node = mask_node_1;
2825 12339 : return true;
2826 : }
2827 :
2828 :
2829 : /* Return true if stored value is suitable for vectorizing store
2830 : statement STMT_INFO. When returning true, store the scalar stored
2831 : in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2832 : the type of the vectorized store value in
2833 : *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2834 :
2835 : static bool
2836 1385417 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
2837 : slp_tree slp_node, slp_tree *rhs_node,
2838 : vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
2839 : vec_load_store_type *vls_type_out)
2840 : {
2841 1385417 : int op_no = 0;
2842 1385417 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
2843 : {
2844 1857 : if (gimple_call_internal_p (call)
2845 1857 : && internal_store_fn_p (gimple_call_internal_fn (call)))
2846 1857 : op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
2847 : }
2848 1385417 : op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
2849 :
2850 1385417 : enum vect_def_type rhs_dt;
2851 1385417 : tree rhs_vectype;
2852 1385417 : tree rhs;
2853 1385417 : if (!vect_is_simple_use (vinfo, slp_node, op_no,
2854 : &rhs, rhs_node, &rhs_dt, &rhs_vectype))
2855 : {
2856 0 : if (dump_enabled_p ())
2857 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2858 : "use not simple.\n");
2859 : return false;
2860 : }
2861 :
2862 : /* In the case this is a store from a constant make sure
2863 : native_encode_expr can handle it. */
2864 1385417 : if (rhs_dt == vect_constant_def
2865 1385417 : && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2866 : {
2867 0 : if (dump_enabled_p ())
2868 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2869 : "cannot encode constant as a byte sequence.\n");
2870 : return false;
2871 : }
2872 :
2873 1385417 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2874 1385417 : if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2875 : {
2876 24 : if (dump_enabled_p ())
2877 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2878 : "incompatible vector types.\n");
2879 : return false;
2880 : }
2881 :
2882 1385393 : *rhs_dt_out = rhs_dt;
2883 1385393 : *rhs_vectype_out = rhs_vectype;
2884 1385393 : if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2885 1027514 : *vls_type_out = VLS_STORE_INVARIANT;
2886 : else
2887 357879 : *vls_type_out = VLS_STORE;
2888 : return true;
2889 : }
2890 :
2891 : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2892 : Note that we support masks with floating-point type, in which case the
2893 : floats are interpreted as a bitmask. */
2894 :
2895 : static tree
2896 155 : vect_build_all_ones_mask (vec_info *vinfo,
2897 : stmt_vec_info stmt_info, tree masktype)
2898 : {
2899 155 : if (TREE_CODE (masktype) == INTEGER_TYPE)
2900 98 : return build_int_cst (masktype, -1);
2901 57 : else if (VECTOR_BOOLEAN_TYPE_P (masktype)
2902 114 : || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2903 : {
2904 8 : tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2905 8 : mask = build_vector_from_val (masktype, mask);
2906 8 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2907 : }
2908 49 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2909 : {
2910 : REAL_VALUE_TYPE r;
2911 : long tmp[6];
2912 343 : for (int j = 0; j < 6; ++j)
2913 294 : tmp[j] = -1;
2914 49 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2915 49 : tree mask = build_real (TREE_TYPE (masktype), r);
2916 49 : mask = build_vector_from_val (masktype, mask);
2917 49 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2918 : }
2919 0 : gcc_unreachable ();
2920 : }
2921 :
2922 : /* Build an all-zero merge value of type VECTYPE while vectorizing
2923 : STMT_INFO as a gather load. */
2924 :
2925 : static tree
2926 154 : vect_build_zero_merge_argument (vec_info *vinfo,
2927 : stmt_vec_info stmt_info, tree vectype)
2928 : {
2929 154 : tree merge;
2930 154 : if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2931 49 : merge = build_int_cst (TREE_TYPE (vectype), 0);
2932 105 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2933 : {
2934 : REAL_VALUE_TYPE r;
2935 : long tmp[6];
2936 735 : for (int j = 0; j < 6; ++j)
2937 630 : tmp[j] = 0;
2938 105 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2939 105 : merge = build_real (TREE_TYPE (vectype), r);
2940 : }
2941 : else
2942 0 : gcc_unreachable ();
2943 154 : merge = build_vector_from_val (vectype, merge);
2944 154 : return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
2945 : }
2946 :
2947 : /* Return the corresponding else value for an else value constant
2948 : ELSVAL with type TYPE. */
2949 :
2950 : tree
2951 1841 : vect_get_mask_load_else (int elsval, tree type)
2952 : {
2953 1841 : tree els;
2954 1841 : if (elsval == MASK_LOAD_ELSE_UNDEFINED)
2955 : {
2956 0 : tree tmp = create_tmp_var (type);
2957 : /* No need to warn about anything. */
2958 0 : TREE_NO_WARNING (tmp) = 1;
2959 0 : els = get_or_create_ssa_default_def (cfun, tmp);
2960 : }
2961 1841 : else if (elsval == MASK_LOAD_ELSE_M1)
2962 0 : els = build_minus_one_cst (type);
2963 1841 : else if (elsval == MASK_LOAD_ELSE_ZERO)
2964 1841 : els = build_zero_cst (type);
2965 : else
2966 0 : gcc_unreachable ();
2967 :
2968 1841 : return els;
2969 : }
2970 :
2971 : /* Build a gather load call while vectorizing STMT_INFO. Insert new
2972 : instructions before GSI and add them to VEC_STMT. GS_INFO describes
2973 : the gather load operation. If the load is conditional, MASK is the
2974 : vectorized condition, otherwise MASK is null. PTR is the base
2975 : pointer and OFFSET is the vectorized offset. */
2976 :
2977 : static gimple *
2978 345 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
2979 : slp_tree slp_node, tree vectype,
2980 : gimple_stmt_iterator *gsi, tree decl,
2981 : tree ptr, tree offset, tree mask)
2982 : {
2983 345 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
2984 345 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
2985 345 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2986 345 : /* ptrtype */ arglist = TREE_CHAIN (arglist);
2987 345 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2988 345 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2989 345 : tree scaletype = TREE_VALUE (arglist);
2990 345 : tree var;
2991 345 : gcc_checking_assert (types_compatible_p (srctype, rettype)
2992 : && (!mask
2993 : || TREE_CODE (masktype) == INTEGER_TYPE
2994 : || types_compatible_p (srctype, masktype)));
2995 :
2996 345 : tree op = offset;
2997 345 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
2998 : {
2999 99 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3000 : TYPE_VECTOR_SUBPARTS (idxtype)));
3001 99 : var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3002 99 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3003 99 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3004 99 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3005 99 : op = var;
3006 : }
3007 :
3008 345 : tree src_op = NULL_TREE;
3009 345 : tree mask_op = NULL_TREE;
3010 345 : if (mask)
3011 : {
3012 191 : if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
3013 : {
3014 191 : tree utype, optype = TREE_TYPE (mask);
3015 191 : if (VECTOR_TYPE_P (masktype)
3016 191 : || TYPE_MODE (masktype) == TYPE_MODE (optype))
3017 : utype = masktype;
3018 : else
3019 6 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3020 191 : var = vect_get_new_ssa_name (utype, vect_scalar_var);
3021 191 : tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
3022 191 : gassign *new_stmt
3023 191 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3024 191 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3025 191 : mask_arg = var;
3026 191 : if (!useless_type_conversion_p (masktype, utype))
3027 : {
3028 6 : gcc_assert (TYPE_PRECISION (utype)
3029 : <= TYPE_PRECISION (masktype));
3030 6 : var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3031 6 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3032 6 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3033 6 : mask_arg = var;
3034 : }
3035 191 : src_op = build_zero_cst (srctype);
3036 191 : mask_op = mask_arg;
3037 : }
3038 : else
3039 : {
3040 : src_op = mask;
3041 : mask_op = mask;
3042 : }
3043 : }
3044 : else
3045 : {
3046 154 : src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
3047 154 : mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
3048 : }
3049 :
3050 345 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3051 345 : gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
3052 : mask_op, scale);
3053 :
3054 345 : if (!useless_type_conversion_p (vectype, rettype))
3055 : {
3056 52 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
3057 : TYPE_VECTOR_SUBPARTS (rettype)));
3058 52 : op = vect_get_new_ssa_name (rettype, vect_simple_var);
3059 52 : gimple_call_set_lhs (new_stmt, op);
3060 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3061 52 : op = build1 (VIEW_CONVERT_EXPR, vectype, op);
3062 52 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
3063 : }
3064 :
3065 345 : return new_stmt;
3066 : }
3067 :
3068 : /* Build a scatter store call while vectorizing STMT_INFO. Insert new
3069 : instructions before GSI. GS_INFO describes the scatter store operation.
3070 : PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
3071 : vectorized data to store.
3072 : If the store is conditional, MASK is the vectorized condition, otherwise
3073 : MASK is null. */
3074 :
3075 : static gimple *
3076 161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
3077 : slp_tree slp_node,
3078 : gimple_stmt_iterator *gsi,
3079 : tree decl,
3080 : tree ptr, tree offset, tree oprnd, tree mask)
3081 : {
3082 161 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
3083 161 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
3084 161 : /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
3085 161 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3086 161 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3087 161 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3088 161 : tree scaletype = TREE_VALUE (arglist);
3089 161 : gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
3090 : && TREE_CODE (rettype) == VOID_TYPE);
3091 :
3092 161 : tree mask_arg = NULL_TREE;
3093 161 : if (mask)
3094 : {
3095 110 : mask_arg = mask;
3096 110 : tree optype = TREE_TYPE (mask_arg);
3097 110 : tree utype;
3098 110 : if (TYPE_MODE (masktype) == TYPE_MODE (optype))
3099 : utype = masktype;
3100 : else
3101 8 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3102 110 : tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
3103 110 : mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
3104 110 : gassign *new_stmt
3105 110 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3106 110 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3107 110 : mask_arg = var;
3108 110 : if (!useless_type_conversion_p (masktype, utype))
3109 : {
3110 8 : gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
3111 8 : tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3112 8 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3113 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3114 8 : mask_arg = var;
3115 : }
3116 : }
3117 : else
3118 : {
3119 51 : mask_arg = build_int_cst (masktype, -1);
3120 51 : mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
3121 : }
3122 :
3123 161 : tree src = oprnd;
3124 161 : if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
3125 : {
3126 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
3127 : TYPE_VECTOR_SUBPARTS (srctype)));
3128 0 : tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
3129 0 : src = build1 (VIEW_CONVERT_EXPR, srctype, src);
3130 0 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
3131 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3132 0 : src = var;
3133 : }
3134 :
3135 161 : tree op = offset;
3136 161 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3137 : {
3138 16 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3139 : TYPE_VECTOR_SUBPARTS (idxtype)));
3140 16 : tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3141 16 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3142 16 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3143 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3144 16 : op = var;
3145 : }
3146 :
3147 161 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3148 161 : gcall *new_stmt
3149 161 : = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
3150 161 : return new_stmt;
3151 : }
3152 :
3153 : /* Prepare the base and offset in GS_INFO for vectorization.
3154 : Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
3155 : to the vectorized offset argument for the first copy of STMT_INFO.
3156 : STMT_INFO is the statement described by GS_INFO and LOOP is the
3157 : containing loop. */
3158 :
3159 : static void
3160 1269 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
3161 : tree *dataref_ptr, vec<tree> *vec_offset)
3162 : {
3163 1269 : gimple_seq stmts = NULL;
3164 1269 : *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
3165 : &stmts, true, NULL_TREE);
3166 1269 : if (stmts != NULL)
3167 : {
3168 1040 : basic_block new_bb;
3169 1040 : edge pe = loop_preheader_edge (loop);
3170 1040 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3171 1040 : gcc_assert (!new_bb);
3172 : }
3173 1269 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
3174 1269 : }
3175 :
3176 : /* Prepare to implement a grouped or strided load or store using
3177 : the gather load or scatter store operation described by GS_INFO.
3178 : STMT_INFO is the load or store statement.
3179 :
3180 : Set *DR_STEP to the amount that should be added to pointer base address
3181 : to get to the next iteration's base address.
3182 : Set *DR_BUMP to the amount that should be added to the base
3183 : address after each copy of the vectorized statement in a grouped read.
3184 : Set *VEC_OFFSET to an invariant offset vector in which element I has the
3185 : value I * DR_STEP / SCALE. */
3186 :
3187 : static void
3188 0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
3189 : tree vectype, tree offset_vectype,
3190 : loop_vec_info loop_vinfo,
3191 : gimple_stmt_iterator *gsi,
3192 : tree *dr_step, tree *dr_bump,
3193 : tree *vec_offset)
3194 : {
3195 0 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
3196 :
3197 0 : tree dr_step_temp
3198 0 : = size_binop (MULT_EXPR,
3199 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3200 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
3201 0 : *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
3202 0 : ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
3203 0 : : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
3204 : true, GSI_SAME_STMT);
3205 0 : tree bump = size_binop (MULT_EXPR,
3206 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3207 : size_int (TYPE_VECTOR_SUBPARTS (vectype)));
3208 0 : *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
3209 :
3210 0 : internal_fn ifn
3211 0 : = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
3212 0 : if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
3213 : {
3214 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
3215 : unshare_expr (DR_STEP (dr)));
3216 0 : return;
3217 : }
3218 :
3219 : /* The offset given in GS_INFO can have pointer type, so use the element
3220 : type of the vector instead. */
3221 0 : tree offset_type = TREE_TYPE (offset_vectype);
3222 :
3223 : /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3224 0 : tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
3225 : ssize_int (SLP_TREE_GS_SCALE (node)));
3226 0 : step = fold_convert (offset_type, step);
3227 :
3228 : /* Create {0, X, X*2, X*3, ...}. */
3229 0 : tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
3230 : build_zero_cst (offset_type), step);
3231 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
3232 : }
3233 :
3234 : /* Return the amount that should be added to a vector pointer, represented by
3235 : DR_INFO, to increment to the next vectorized iteration. */
3236 :
3237 : static tree
3238 697202 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
3239 : vect_memory_access_type memory_access_type)
3240 : {
3241 697202 : if (memory_access_type == VMAT_INVARIANT)
3242 0 : return size_zero_node;
3243 :
3244 697202 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3245 :
3246 : /* For BB SLP there is no next iteration. */
3247 697202 : if (!loop_vinfo)
3248 559275 : return build_zero_cst (sizetype);
3249 :
3250 137927 : tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
3251 :
3252 : /* gather/scatter never reach here. */
3253 137927 : gcc_assert (!mat_gather_scatter_p (memory_access_type));
3254 :
3255 137927 : tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
3256 :
3257 137927 : return fold_build2 (MULT_EXPR, sizetype, iv_increment,
3258 : fold_convert (sizetype, step));
3259 : }
3260 :
3261 : /* Return the amount that should be added to a vector pointer to move
3262 : to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3263 : being vectorized and MEMORY_ACCESS_TYPE describes the type of
3264 : vectorization. */
3265 :
3266 : static tree
3267 697714 : vect_get_data_ptr_bump (vec_info *vinfo,
3268 : dr_vec_info *dr_info, tree aggr_type,
3269 : vect_memory_access_type memory_access_type)
3270 : {
3271 697714 : if (memory_access_type == VMAT_INVARIANT)
3272 0 : return size_zero_node;
3273 :
3274 697714 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3275 : /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
3276 : should never be needed. */
3277 138439 : if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
3278 : return NULL_TREE;
3279 :
3280 697714 : tree iv_step = TYPE_SIZE_UNIT (aggr_type);
3281 697714 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
3282 697714 : if (tree_int_cst_sgn (step) == -1)
3283 2849 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
3284 : return iv_step;
3285 : }
3286 :
3287 : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3288 :
3289 : static bool
3290 156 : vectorizable_bswap (vec_info *vinfo,
3291 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3292 : slp_tree slp_node,
3293 : slp_tree *slp_op,
3294 : tree vectype_in, stmt_vector_for_cost *cost_vec)
3295 : {
3296 156 : tree vectype = SLP_TREE_VECTYPE (slp_node);
3297 156 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3298 :
3299 156 : if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
3300 : {
3301 0 : if (dump_enabled_p ())
3302 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3303 : "mismatched vector sizes %T and %T\n",
3304 : vectype_in, vectype);
3305 : return false;
3306 : }
3307 :
3308 156 : tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
3309 156 : if (! char_vectype)
3310 : return false;
3311 :
3312 156 : poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
3313 156 : unsigned word_bytes;
3314 156 : if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
3315 : return false;
3316 :
3317 : /* The encoding uses one stepped pattern for each byte in the word. */
3318 156 : vec_perm_builder elts (num_bytes, word_bytes, 3);
3319 780 : for (unsigned i = 0; i < 3; ++i)
3320 2772 : for (unsigned j = 0; j < word_bytes; ++j)
3321 2304 : elts.quick_push ((i + 1) * word_bytes - j - 1);
3322 :
3323 156 : vec_perm_indices indices (elts, 1, num_bytes);
3324 156 : machine_mode vmode = TYPE_MODE (char_vectype);
3325 156 : if (!can_vec_perm_const_p (vmode, vmode, indices))
3326 : return false;
3327 :
3328 61 : if (cost_vec)
3329 : {
3330 45 : if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
3331 : {
3332 0 : if (dump_enabled_p ())
3333 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3334 : "incompatible vector types for invariants\n");
3335 : return false;
3336 : }
3337 :
3338 45 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3339 45 : DUMP_VECT_SCOPE ("vectorizable_bswap");
3340 45 : record_stmt_cost (cost_vec,
3341 : 1, vector_stmt, slp_node, 0, vect_prologue);
3342 45 : record_stmt_cost (cost_vec,
3343 45 : vect_get_num_copies (vinfo, slp_node),
3344 : vec_perm, slp_node, 0, vect_body);
3345 45 : return true;
3346 : }
3347 :
3348 16 : tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
3349 :
3350 : /* Transform. */
3351 16 : vec<tree> vec_oprnds = vNULL;
3352 16 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
3353 : /* Arguments are ready. create the new vector stmt. */
3354 16 : unsigned i;
3355 16 : tree vop;
3356 48 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
3357 : {
3358 16 : gimple *new_stmt;
3359 16 : tree tem = make_ssa_name (char_vectype);
3360 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3361 : char_vectype, vop));
3362 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3363 16 : tree tem2 = make_ssa_name (char_vectype);
3364 16 : new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
3365 : tem, tem, bswap_vconst);
3366 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3367 16 : tem = make_ssa_name (vectype);
3368 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3369 : vectype, tem2));
3370 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3371 16 : slp_node->push_vec_def (new_stmt);
3372 : }
3373 :
3374 16 : vec_oprnds.release ();
3375 16 : return true;
3376 156 : }
3377 :
3378 : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3379 : integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3380 : in a single step. On success, store the binary pack code in
3381 : *CONVERT_CODE. */
3382 :
3383 : static bool
3384 191 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
3385 : code_helper *convert_code)
3386 : {
3387 382 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
3388 382 : || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
3389 : return false;
3390 :
3391 81 : code_helper code;
3392 81 : int multi_step_cvt = 0;
3393 81 : auto_vec <tree, 8> interm_types;
3394 121 : if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
3395 : &code, &multi_step_cvt, &interm_types)
3396 81 : || multi_step_cvt)
3397 : return false;
3398 :
3399 41 : *convert_code = code;
3400 41 : return true;
3401 81 : }
3402 :
3403 : /* Function vectorizable_call.
3404 :
3405 : Check if STMT_INFO performs a function call that can be vectorized.
3406 : If COST_VEC is passed, calculate costs but don't change anything,
3407 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
3408 : it, and insert it at GSI.
3409 : Return true if STMT_INFO is vectorizable in this way. */
3410 :
3411 : static bool
3412 2665155 : vectorizable_call (vec_info *vinfo,
3413 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3414 : slp_tree slp_node,
3415 : stmt_vector_for_cost *cost_vec)
3416 : {
3417 2665155 : gcall *stmt;
3418 2665155 : tree vec_dest;
3419 2665155 : tree scalar_dest;
3420 2665155 : tree vec_oprnd0 = NULL_TREE;
3421 2665155 : tree vectype_out, vectype_in;
3422 2665155 : poly_uint64 nunits_in;
3423 2665155 : poly_uint64 nunits_out;
3424 2665155 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3425 2665155 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3426 2665155 : tree fndecl, new_temp, rhs_type;
3427 2665155 : enum vect_def_type dt[5]
3428 : = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
3429 : vect_unknown_def_type, vect_unknown_def_type };
3430 2665155 : tree vectypes[ARRAY_SIZE (dt)] = {};
3431 2665155 : slp_tree slp_op[ARRAY_SIZE (dt)] = {};
3432 2665155 : auto_vec<tree, 8> vargs;
3433 2665155 : enum { NARROW, NONE, WIDEN } modifier;
3434 2665155 : size_t i, nargs;
3435 2665155 : tree clz_ctz_arg1 = NULL_TREE;
3436 :
3437 2665155 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3438 : return false;
3439 :
3440 2665155 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3441 247311 : && cost_vec)
3442 : return false;
3443 :
3444 : /* Is STMT_INFO a vectorizable call? */
3445 2691650 : stmt = dyn_cast <gcall *> (stmt_info->stmt);
3446 26495 : if (!stmt)
3447 : return false;
3448 :
3449 26495 : if (gimple_call_internal_p (stmt)
3450 26495 : && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3451 17197 : || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3452 : /* Handled by vectorizable_load and vectorizable_store. */
3453 : return false;
3454 :
3455 23013 : if (gimple_call_lhs (stmt) == NULL_TREE
3456 23013 : || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3457 : return false;
3458 :
3459 23007 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
3460 :
3461 23007 : vectype_out = SLP_TREE_VECTYPE (slp_node);
3462 :
3463 : /* Process function arguments. */
3464 23007 : rhs_type = NULL_TREE;
3465 23007 : vectype_in = NULL_TREE;
3466 23007 : nargs = gimple_call_num_args (stmt);
3467 :
3468 : /* Bail out if the function has more than four arguments, we do not have
3469 : interesting builtin functions to vectorize with more than two arguments
3470 : except for fma (cond_fma has more). No arguments is also not good. */
3471 23007 : if (nargs == 0 || nargs > 5)
3472 : return false;
3473 :
3474 : /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3475 22927 : combined_fn cfn = gimple_call_combined_fn (stmt);
3476 22927 : if (cfn == CFN_GOMP_SIMD_LANE)
3477 : {
3478 3205 : nargs = 0;
3479 3205 : rhs_type = unsigned_type_node;
3480 : }
3481 : /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3482 : argument just says whether it is well-defined at zero or not and what
3483 : value should be returned for it. */
3484 22927 : if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
3485 : {
3486 168 : nargs = 1;
3487 168 : clz_ctz_arg1 = gimple_call_arg (stmt, 1);
3488 : }
3489 :
3490 22927 : int mask_opno = -1;
3491 22927 : if (internal_fn_p (cfn))
3492 : {
3493 : /* We can only handle direct internal masked calls here,
3494 : vectorizable_simd_clone_call is for the rest. */
3495 19454 : if (cfn == CFN_MASK_CALL)
3496 : return false;
3497 19300 : mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
3498 : }
3499 :
3500 71087 : for (i = 0; i < nargs; i++)
3501 : {
3502 50378 : if ((int) i == mask_opno)
3503 : {
3504 7702 : if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
3505 : &slp_op[i], &dt[i], &vectypes[i]))
3506 : return false;
3507 7702 : continue;
3508 : }
3509 :
3510 42676 : if (!vect_is_simple_use (vinfo, slp_node,
3511 : i, &slp_op[i], &dt[i], &vectypes[i]))
3512 : {
3513 0 : if (dump_enabled_p ())
3514 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3515 : "use not simple.\n");
3516 : return false;
3517 : }
3518 :
3519 : /* We can only handle calls with arguments of the same type. */
3520 42676 : tree op = gimple_call_arg (stmt, i);
3521 42676 : if (rhs_type
3522 42676 : && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3523 : {
3524 2064 : if (dump_enabled_p ())
3525 222 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3526 : "argument types differ.\n");
3527 : return false;
3528 : }
3529 40612 : if (!rhs_type)
3530 19568 : rhs_type = TREE_TYPE (op);
3531 :
3532 40612 : if (!vectype_in)
3533 20854 : vectype_in = vectypes[i];
3534 19758 : else if (vectypes[i]
3535 19758 : && !types_compatible_p (vectypes[i], vectype_in))
3536 : {
3537 0 : if (dump_enabled_p ())
3538 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3539 : "argument vector types differ.\n");
3540 : return false;
3541 : }
3542 : }
3543 : /* If all arguments are external or constant defs, infer the vector type
3544 : from the scalar type. */
3545 20709 : if (!vectype_in)
3546 5814 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
3547 20709 : if (!cost_vec)
3548 4328 : gcc_assert (vectype_in);
3549 16381 : if (!vectype_in)
3550 : {
3551 1098 : if (dump_enabled_p ())
3552 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3553 : "no vectype for scalar type %T\n", rhs_type);
3554 :
3555 : return false;
3556 : }
3557 :
3558 : /* During vect-patterns we remembered the defined-at-zero value in the
3559 : optional second argument for the natural vector mode. Now, once we have
3560 : the final vector mode, verify that they coincide. If not, bail out. */
3561 19611 : if (clz_ctz_arg1)
3562 : {
3563 168 : int val;
3564 168 : int d = (cfn == CFN_CLZ
3565 168 : ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val)
3566 168 : : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val));
3567 168 : if (d != 2 || val != tree_to_shwi (clz_ctz_arg1))
3568 : return false;
3569 : }
3570 :
3571 39222 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3572 19611 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
3573 : {
3574 12 : if (dump_enabled_p ())
3575 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3576 : "mixed mask and nonmask vector types\n");
3577 : return false;
3578 : }
3579 :
3580 19599 : if (vect_emulated_vector_p (vectype_in)
3581 19599 : || vect_emulated_vector_p (vectype_out))
3582 : {
3583 0 : if (dump_enabled_p ())
3584 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3585 : "use emulated vector type for call\n");
3586 : return false;
3587 : }
3588 :
3589 : /* FORNOW */
3590 19599 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3591 19599 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3592 19599 : if (known_eq (nunits_in * 2, nunits_out))
3593 : modifier = NARROW;
3594 19017 : else if (known_eq (nunits_out, nunits_in))
3595 : modifier = NONE;
3596 50 : else if (known_eq (nunits_out * 2, nunits_in))
3597 : modifier = WIDEN;
3598 : else
3599 : return false;
3600 :
3601 : /* We only handle functions that do not read or clobber memory. */
3602 39198 : if (gimple_vuse (stmt))
3603 : {
3604 1421 : if (dump_enabled_p ())
3605 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3606 : "function reads from or writes to memory.\n");
3607 : return false;
3608 : }
3609 :
3610 : /* For now, we only vectorize functions if a target specific builtin
3611 : is available. TODO -- in some cases, it might be profitable to
3612 : insert the calls for pieces of the vector, in order to be able
3613 : to vectorize other operations in the loop. */
3614 18178 : fndecl = NULL_TREE;
3615 18178 : internal_fn ifn = IFN_LAST;
3616 18178 : tree callee = gimple_call_fndecl (stmt);
3617 :
3618 : /* First try using an internal function. */
3619 18178 : code_helper convert_code = MAX_TREE_CODES;
3620 18178 : if (cfn != CFN_LAST
3621 18178 : && (modifier == NONE
3622 203 : || (modifier == NARROW
3623 191 : && simple_integer_narrowing (vectype_out, vectype_in,
3624 : &convert_code))))
3625 17137 : ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3626 : vectype_in);
3627 :
3628 : /* Check if the operation traps. */
3629 18178 : bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
3630 18178 : if (could_trap && cost_vec && loop_vinfo)
3631 : {
3632 : /* If the operation can trap it must be conditional, otherwise fail. */
3633 546 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3634 546 : ? ifn : get_conditional_internal_fn (ifn));
3635 546 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3636 546 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
3637 : {
3638 : /* We assume that BB SLP fills all lanes, so no inactive lanes can
3639 : cause issues. */
3640 124 : if ((cond_fn == IFN_LAST
3641 76 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3642 : OPTIMIZE_FOR_SPEED))
3643 200 : && (cond_len_fn == IFN_LAST
3644 76 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3645 : OPTIMIZE_FOR_SPEED)))
3646 : {
3647 124 : if (dump_enabled_p ())
3648 10 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3649 : "can't use a fully-masked loop because no"
3650 : " conditional operation is available.\n");
3651 124 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3652 : }
3653 : }
3654 : }
3655 :
3656 : /* If that fails, try asking for a target-specific built-in function. */
3657 18178 : if (ifn == IFN_LAST)
3658 : {
3659 9994 : if (cfn != CFN_LAST)
3660 9115 : fndecl = targetm.vectorize.builtin_vectorized_function
3661 9115 : (cfn, vectype_out, vectype_in);
3662 879 : else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
3663 24 : fndecl = targetm.vectorize.builtin_md_vectorized_function
3664 24 : (callee, vectype_out, vectype_in);
3665 : }
3666 :
3667 18178 : if (ifn == IFN_LAST && !fndecl)
3668 : {
3669 9614 : if (cfn == CFN_GOMP_SIMD_LANE
3670 3205 : && SLP_TREE_LANES (slp_node) == 1
3671 3205 : && loop_vinfo
3672 3205 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3673 3205 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3674 16024 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3675 3205 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3676 : {
3677 : /* We can handle IFN_GOMP_SIMD_LANE by returning a
3678 : { 0, 1, 2, ... vf - 1 } vector. */
3679 3205 : gcc_assert (nargs == 0);
3680 : }
3681 6409 : else if (modifier == NONE
3682 6409 : && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3683 6052 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3684 5983 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
3685 5930 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
3686 156 : return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
3687 156 : slp_op, vectype_in, cost_vec);
3688 : else
3689 : {
3690 6253 : if (dump_enabled_p ())
3691 276 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3692 : "function is not vectorizable.\n");
3693 : return false;
3694 : }
3695 : }
3696 :
3697 11769 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
3698 11769 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3699 11769 : ? ifn : get_conditional_internal_fn (ifn));
3700 11769 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3701 11769 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3702 9646 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
3703 11769 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
3704 11769 : if (cost_vec) /* transformation not required. */
3705 : {
3706 23491 : for (i = 0; i < nargs; ++i)
3707 16034 : if (!vect_maybe_update_slp_op_vectype (slp_op[i],
3708 16034 : vectypes[i]
3709 : ? vectypes[i] : vectype_in))
3710 : {
3711 0 : if (dump_enabled_p ())
3712 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3713 : "incompatible vector types for invariants\n");
3714 : return false;
3715 : }
3716 7457 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3717 7457 : DUMP_VECT_SCOPE ("vectorizable_call");
3718 7457 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
3719 :
3720 7457 : if (loop_vinfo
3721 6274 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
3722 4115 : && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3723 : {
3724 2562 : if (reduc_idx >= 0
3725 1631 : && (cond_fn == IFN_LAST
3726 1631 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3727 : OPTIMIZE_FOR_SPEED))
3728 2574 : && (cond_len_fn == IFN_LAST
3729 12 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3730 : OPTIMIZE_FOR_SPEED)))
3731 : {
3732 12 : if (dump_enabled_p ())
3733 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3734 : "can't use a fully-masked loop because no"
3735 : " conditional operation is available.\n");
3736 12 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3737 : }
3738 : else
3739 : {
3740 2550 : tree scalar_mask = NULL_TREE;
3741 2550 : if (mask_opno >= 0)
3742 2550 : scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3743 2550 : if (cond_len_fn != IFN_LAST
3744 2550 : && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3745 : OPTIMIZE_FOR_SPEED))
3746 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3747 : 1);
3748 : else
3749 2550 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3750 : scalar_mask);
3751 : }
3752 : }
3753 7457 : return true;
3754 : }
3755 :
3756 : /* Transform. */
3757 :
3758 4312 : if (dump_enabled_p ())
3759 429 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3760 :
3761 : /* Handle def. */
3762 4312 : scalar_dest = gimple_call_lhs (stmt);
3763 4312 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3764 :
3765 4312 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3766 3372 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
3767 4312 : unsigned int vect_nargs = nargs;
3768 4312 : if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3769 : {
3770 0 : ifn = cond_len_fn;
3771 : /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3772 0 : vect_nargs += 2;
3773 : /* But unless there's a mask argument already we need that
3774 : as well, and an else value. */
3775 0 : if (mask_opno == -1)
3776 0 : vect_nargs += 2;
3777 : }
3778 4312 : else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
3779 : {
3780 0 : ifn = cond_fn;
3781 0 : vect_nargs += 2;
3782 : }
3783 4312 : int len_opno = internal_fn_len_index (ifn);
3784 4312 : if (clz_ctz_arg1)
3785 59 : ++vect_nargs;
3786 :
3787 4312 : if (modifier == NONE || ifn != IFN_LAST)
3788 : {
3789 4280 : tree prev_res = NULL_TREE;
3790 4280 : vargs.safe_grow (vect_nargs, true);
3791 4280 : auto_vec<vec<tree> > vec_defs (nargs);
3792 :
3793 : /* Build argument list for the vectorized call. */
3794 4280 : if (cfn == CFN_GOMP_SIMD_LANE)
3795 : {
3796 3306 : for (i = 0; i < nvectors; ++i)
3797 : {
3798 : /* ??? For multi-lane SLP we'd need to build
3799 : { 0, 0, .., 1, 1, ... }. */
3800 1707 : tree cst = build_index_vector (vectype_out,
3801 : i * nunits_out, 1);
3802 1707 : tree new_var
3803 1707 : = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3804 1707 : gimple *init_stmt = gimple_build_assign (new_var, cst);
3805 1707 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
3806 1707 : new_temp = make_ssa_name (vec_dest);
3807 1707 : gimple *new_stmt = gimple_build_assign (new_temp, new_var);
3808 1707 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3809 1707 : slp_node->push_vec_def (new_stmt);
3810 : }
3811 : }
3812 : else
3813 : {
3814 2681 : vec<tree> vec_oprnds0;
3815 2681 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3816 2681 : vec_oprnds0 = vec_defs[0];
3817 :
3818 : /* Arguments are ready. Create the new vector stmt. */
3819 5513 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3820 : {
3821 2832 : int varg = 0;
3822 : /* Add the mask if necessary. */
3823 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3824 2834 : && internal_fn_mask_index (ifn) != -1)
3825 : {
3826 0 : gcc_assert (internal_fn_mask_index (ifn) == varg);
3827 0 : if (masked_loop_p)
3828 : {
3829 0 : unsigned int vec_num = vec_oprnds0.length ();
3830 0 : vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
3831 : masks, vec_num,
3832 : vectype_out, i);
3833 : }
3834 : else
3835 : {
3836 0 : tree mask_vectype = truth_type_for (vectype_out);
3837 0 : vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
3838 : stmt_info,
3839 : mask_vectype);
3840 : }
3841 : }
3842 2832 : size_t k;
3843 10255 : for (k = 0; k < nargs; k++)
3844 : {
3845 7423 : vec<tree> vec_oprndsk = vec_defs[k];
3846 7423 : vargs[varg++] = vec_oprndsk[i];
3847 : }
3848 : /* Add the else value if necessary. */
3849 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3850 2834 : && internal_fn_else_index (ifn) != -1)
3851 : {
3852 0 : gcc_assert (internal_fn_else_index (ifn) == varg);
3853 0 : if (reduc_idx >= 0)
3854 0 : vargs[varg++] = vargs[reduc_idx + 1];
3855 : else
3856 : {
3857 0 : auto else_value = targetm.preferred_else_value
3858 0 : (ifn, vectype_out, varg - 1, &vargs[1]);
3859 0 : vargs[varg++] = else_value;
3860 : }
3861 : }
3862 2832 : if (clz_ctz_arg1)
3863 59 : vargs[varg++] = clz_ctz_arg1;
3864 :
3865 2832 : gimple *new_stmt;
3866 2832 : if (modifier == NARROW)
3867 : {
3868 : /* We don't define any narrowing conditional functions
3869 : at present. */
3870 0 : gcc_assert (mask_opno < 0);
3871 0 : tree half_res = make_ssa_name (vectype_in);
3872 0 : gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3873 0 : gimple_call_set_lhs (call, half_res);
3874 0 : gimple_call_set_nothrow (call, true);
3875 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3876 0 : if ((i & 1) == 0)
3877 : {
3878 0 : prev_res = half_res;
3879 0 : continue;
3880 : }
3881 0 : new_temp = make_ssa_name (vec_dest);
3882 0 : new_stmt = vect_gimple_build (new_temp, convert_code,
3883 : prev_res, half_res);
3884 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3885 : }
3886 : else
3887 : {
3888 2832 : if (len_opno >= 0 && len_loop_p)
3889 : {
3890 0 : unsigned int vec_num = vec_oprnds0.length ();
3891 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
3892 : vec_num, vectype_out, i, 1, true);
3893 0 : signed char biasval
3894 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3895 0 : tree bias = build_int_cst (intQI_type_node, biasval);
3896 0 : vargs[len_opno] = len;
3897 0 : vargs[len_opno + 1] = bias;
3898 : }
3899 2832 : else if (mask_opno >= 0 && masked_loop_p)
3900 : {
3901 36 : unsigned int vec_num = vec_oprnds0.length ();
3902 36 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
3903 : vec_num, vectype_out, i);
3904 36 : vargs[mask_opno]
3905 72 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
3906 36 : vargs[mask_opno], gsi);
3907 : }
3908 :
3909 2832 : gcall *call;
3910 2832 : if (ifn != IFN_LAST)
3911 2751 : call = gimple_build_call_internal_vec (ifn, vargs);
3912 : else
3913 81 : call = gimple_build_call_vec (fndecl, vargs);
3914 2832 : new_temp = make_ssa_name (vec_dest, call);
3915 2832 : gimple_call_set_lhs (call, new_temp);
3916 2832 : gimple_call_set_nothrow (call, true);
3917 2832 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3918 2832 : new_stmt = call;
3919 : }
3920 2832 : slp_node->push_vec_def (new_stmt);
3921 : }
3922 : }
3923 :
3924 11349 : for (i = 0; i < nargs; i++)
3925 : {
3926 7069 : vec<tree> vec_oprndsi = vec_defs[i];
3927 7069 : vec_oprndsi.release ();
3928 : }
3929 4280 : }
3930 32 : else if (modifier == NARROW)
3931 : {
3932 32 : auto_vec<vec<tree> > vec_defs (nargs);
3933 : /* We don't define any narrowing conditional functions at present. */
3934 32 : gcc_assert (mask_opno < 0);
3935 :
3936 : /* Build argument list for the vectorized call. */
3937 32 : vargs.create (nargs * 2);
3938 :
3939 32 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3940 32 : vec<tree> vec_oprnds0 = vec_defs[0];
3941 :
3942 : /* Arguments are ready. Create the new vector stmt. */
3943 64 : for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3944 : {
3945 32 : size_t k;
3946 32 : vargs.truncate (0);
3947 96 : for (k = 0; k < nargs; k++)
3948 : {
3949 32 : vec<tree> vec_oprndsk = vec_defs[k];
3950 32 : vargs.quick_push (vec_oprndsk[i]);
3951 32 : vargs.quick_push (vec_oprndsk[i + 1]);
3952 : }
3953 32 : gcall *call;
3954 32 : if (ifn != IFN_LAST)
3955 : call = gimple_build_call_internal_vec (ifn, vargs);
3956 : else
3957 32 : call = gimple_build_call_vec (fndecl, vargs);
3958 32 : new_temp = make_ssa_name (vec_dest, call);
3959 32 : gimple_call_set_lhs (call, new_temp);
3960 32 : gimple_call_set_nothrow (call, true);
3961 32 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3962 32 : slp_node->push_vec_def (call);
3963 : }
3964 :
3965 64 : for (i = 0; i < nargs; i++)
3966 : {
3967 32 : vec<tree> vec_oprndsi = vec_defs[i];
3968 32 : vec_oprndsi.release ();
3969 : }
3970 32 : }
3971 : else
3972 : /* No current target implements this case. */
3973 : return false;
3974 :
3975 4312 : vargs.release ();
3976 :
3977 4312 : return true;
3978 2665155 : }
3979 :
3980 :
3981 : struct simd_call_arg_info
3982 : {
3983 : tree vectype;
3984 : tree op;
3985 : HOST_WIDE_INT linear_step;
3986 : enum vect_def_type dt;
3987 : unsigned int align;
3988 : bool simd_lane_linear;
3989 : };
3990 :
3991 : /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3992 : is linear within simd lane (but not within whole loop), note it in
3993 : *ARGINFO. */
3994 :
3995 : static void
3996 15 : vect_simd_lane_linear (tree op, class loop *loop,
3997 : struct simd_call_arg_info *arginfo)
3998 : {
3999 15 : gimple *def_stmt = SSA_NAME_DEF_STMT (op);
4000 :
4001 15 : if (!is_gimple_assign (def_stmt)
4002 15 : || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
4003 27 : || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
4004 : return;
4005 :
4006 12 : tree base = gimple_assign_rhs1 (def_stmt);
4007 12 : HOST_WIDE_INT linear_step = 0;
4008 12 : tree v = gimple_assign_rhs2 (def_stmt);
4009 48 : while (TREE_CODE (v) == SSA_NAME)
4010 : {
4011 36 : tree t;
4012 36 : def_stmt = SSA_NAME_DEF_STMT (v);
4013 36 : if (is_gimple_assign (def_stmt))
4014 24 : switch (gimple_assign_rhs_code (def_stmt))
4015 : {
4016 0 : case PLUS_EXPR:
4017 0 : t = gimple_assign_rhs2 (def_stmt);
4018 0 : if (linear_step || TREE_CODE (t) != INTEGER_CST)
4019 : return;
4020 0 : base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
4021 0 : v = gimple_assign_rhs1 (def_stmt);
4022 0 : continue;
4023 12 : case MULT_EXPR:
4024 12 : t = gimple_assign_rhs2 (def_stmt);
4025 12 : if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
4026 : return;
4027 12 : linear_step = tree_to_shwi (t);
4028 12 : v = gimple_assign_rhs1 (def_stmt);
4029 12 : continue;
4030 12 : CASE_CONVERT:
4031 12 : t = gimple_assign_rhs1 (def_stmt);
4032 12 : if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
4033 12 : || (TYPE_PRECISION (TREE_TYPE (v))
4034 12 : < TYPE_PRECISION (TREE_TYPE (t))))
4035 : return;
4036 12 : if (!linear_step)
4037 0 : linear_step = 1;
4038 12 : v = t;
4039 12 : continue;
4040 : default:
4041 : return;
4042 : }
4043 12 : else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
4044 12 : && loop->simduid
4045 12 : && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
4046 24 : && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
4047 : == loop->simduid))
4048 : {
4049 12 : if (!linear_step)
4050 0 : linear_step = 1;
4051 12 : arginfo->linear_step = linear_step;
4052 12 : arginfo->op = base;
4053 12 : arginfo->simd_lane_linear = true;
4054 12 : return;
4055 : }
4056 : }
4057 : }
4058 :
4059 : /* Function vectorizable_simd_clone_call.
4060 :
4061 : Check if STMT_INFO performs a function call that can be vectorized
4062 : by calling a simd clone of the function.
4063 : If COST_VEC is passed, calculate costs but don't change anything,
4064 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
4065 : it, and insert it at GSI.
4066 : Return true if STMT_INFO is vectorizable in this way. */
4067 :
4068 : static bool
4069 2653695 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
4070 : gimple_stmt_iterator *gsi,
4071 : slp_tree slp_node,
4072 : stmt_vector_for_cost *cost_vec)
4073 : {
4074 2653695 : tree vec_dest;
4075 2653695 : tree scalar_dest;
4076 2653695 : tree vec_oprnd0 = NULL_TREE;
4077 2653695 : tree vectype;
4078 2653695 : poly_uint64 nunits;
4079 2653695 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
4080 2653695 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
4081 2653695 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
4082 2653695 : tree fndecl, new_temp;
4083 2653695 : int j;
4084 2653695 : auto_vec<simd_call_arg_info> arginfo;
4085 2653695 : vec<tree> vargs = vNULL;
4086 2653695 : size_t i, nargs;
4087 2653695 : tree rtype, ratype;
4088 2653695 : vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
4089 2653695 : int masked_call_offset = 0;
4090 :
4091 : /* Is STMT a vectorizable call? */
4092 2653695 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
4093 16216 : if (!stmt)
4094 : return false;
4095 :
4096 16216 : fndecl = gimple_call_fndecl (stmt);
4097 16216 : if (fndecl == NULL_TREE
4098 16216 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
4099 : {
4100 220 : fndecl = gimple_call_arg (stmt, 0);
4101 220 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
4102 220 : fndecl = TREE_OPERAND (fndecl, 0);
4103 220 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
4104 : masked_call_offset = 1;
4105 : }
4106 15996 : if (fndecl == NULL_TREE)
4107 : return false;
4108 :
4109 6023 : struct cgraph_node *node = cgraph_node::get (fndecl);
4110 6023 : if (node == NULL || node->simd_clones == NULL)
4111 : return false;
4112 :
4113 1506 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4114 : return false;
4115 :
4116 1506 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4117 0 : && cost_vec)
4118 : return false;
4119 :
4120 1506 : if (gimple_call_lhs (stmt)
4121 1506 : && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
4122 : return false;
4123 :
4124 1506 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
4125 :
4126 1506 : vectype = SLP_TREE_VECTYPE (slp_node);
4127 :
4128 2655201 : if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
4129 : return false;
4130 :
4131 : /* Process function arguments. */
4132 1506 : nargs = gimple_call_num_args (stmt) - masked_call_offset;
4133 :
4134 : /* Bail out if the function has zero arguments. */
4135 1506 : if (nargs == 0)
4136 : return false;
4137 :
4138 1442 : vect_simd_clone_data _data;
4139 1442 : vect_simd_clone_data &data = slp_node->get_data (_data);
4140 1442 : vec<tree>& simd_clone_info = data.simd_clone_info;
4141 1442 : arginfo.reserve (nargs, true);
4142 1442 : auto_vec<slp_tree> slp_op;
4143 1442 : slp_op.safe_grow_cleared (nargs);
4144 :
4145 5623 : for (i = 0; i < nargs; i++)
4146 : {
4147 2739 : simd_call_arg_info thisarginfo;
4148 2739 : affine_iv iv;
4149 2739 : tree op;
4150 :
4151 2739 : thisarginfo.linear_step = 0;
4152 2739 : thisarginfo.align = 0;
4153 2739 : thisarginfo.op = NULL_TREE;
4154 2739 : thisarginfo.simd_lane_linear = false;
4155 :
4156 5478 : int op_no = vect_slp_child_index_for_operand (stmt_info,
4157 2739 : i + masked_call_offset);
4158 5478 : if (!vect_is_simple_use (vinfo, slp_node,
4159 2739 : op_no, &op, &slp_op[i],
4160 : &thisarginfo.dt, &thisarginfo.vectype)
4161 2739 : || thisarginfo.dt == vect_uninitialized_def)
4162 : {
4163 0 : if (dump_enabled_p ())
4164 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4165 : "use not simple.\n");
4166 0 : return false;
4167 : }
4168 :
4169 2739 : if (thisarginfo.dt == vect_constant_def
4170 2739 : || thisarginfo.dt == vect_external_def)
4171 : {
4172 : /* With SLP we determine the vector type of constants/externals
4173 : at analysis time, handling conflicts via
4174 : vect_maybe_update_slp_op_vectype. At transform time
4175 : we have a vector type recorded for SLP. */
4176 680 : gcc_assert (cost_vec
4177 : || thisarginfo.vectype != NULL_TREE);
4178 : if (cost_vec)
4179 549 : thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
4180 549 : TREE_TYPE (op),
4181 : slp_node);
4182 : }
4183 : else
4184 2059 : gcc_assert (thisarginfo.vectype != NULL_TREE);
4185 :
4186 : /* For linear arguments, the analyze phase should have saved
4187 : the base and step. */
4188 2608 : if (!cost_vec
4189 1634 : && i * 3 + 4 <= simd_clone_info.length ()
4190 2818 : && simd_clone_info[i * 3 + 2])
4191 : {
4192 118 : thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
4193 118 : thisarginfo.op = simd_clone_info[i * 3 + 1];
4194 118 : thisarginfo.simd_lane_linear
4195 118 : = (simd_clone_info[i * 3 + 3] == boolean_true_node);
4196 : /* If loop has been peeled for alignment, we need to adjust it. */
4197 118 : tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
4198 118 : tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
4199 118 : if (n1 != n2 && !thisarginfo.simd_lane_linear)
4200 : {
4201 0 : tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
4202 0 : tree step = simd_clone_info[i * 3 + 2];
4203 0 : tree opt = TREE_TYPE (thisarginfo.op);
4204 0 : bias = fold_convert (TREE_TYPE (step), bias);
4205 0 : bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
4206 0 : thisarginfo.op
4207 0 : = fold_build2 (POINTER_TYPE_P (opt)
4208 : ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
4209 : thisarginfo.op, bias);
4210 : }
4211 : }
4212 2621 : else if (cost_vec
4213 1922 : && thisarginfo.dt != vect_constant_def
4214 1795 : && thisarginfo.dt != vect_external_def
4215 1373 : && loop_vinfo
4216 1356 : && SLP_TREE_LANES (slp_node) == 1
4217 1332 : && TREE_CODE (op) == SSA_NAME
4218 2664 : && simple_iv (loop, loop_containing_stmt (stmt), op,
4219 : &iv, false)
4220 2833 : && tree_fits_shwi_p (iv.step))
4221 : {
4222 212 : thisarginfo.linear_step = tree_to_shwi (iv.step);
4223 212 : thisarginfo.op = iv.base;
4224 : }
4225 2409 : else if ((thisarginfo.dt == vect_constant_def
4226 2409 : || thisarginfo.dt == vect_external_def)
4227 680 : && SLP_TREE_LANES (slp_node) == 1
4228 2715 : && POINTER_TYPE_P (TREE_TYPE (op)))
4229 86 : thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
4230 : /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4231 : linear too. */
4232 2739 : if (SLP_TREE_LANES (slp_node) == 1
4233 2295 : && POINTER_TYPE_P (TREE_TYPE (op))
4234 196 : && !thisarginfo.linear_step
4235 112 : && cost_vec
4236 58 : && thisarginfo.dt != vect_constant_def
4237 58 : && thisarginfo.dt != vect_external_def
4238 15 : && loop_vinfo
4239 2754 : && TREE_CODE (op) == SSA_NAME)
4240 15 : vect_simd_lane_linear (op, loop, &thisarginfo);
4241 :
4242 2739 : if (!vectype)
4243 12 : vectype = thisarginfo.vectype;
4244 2739 : arginfo.quick_push (thisarginfo);
4245 : }
4246 :
4247 1442 : poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
4248 1442 : unsigned group_size = SLP_TREE_LANES (slp_node);
4249 1442 : unsigned int badness = 0;
4250 1442 : unsigned int badness_inbranch = 0;
4251 1442 : struct cgraph_node *bestn = NULL;
4252 1442 : struct cgraph_node *bestn_inbranch = NULL;
4253 1442 : if (!cost_vec)
4254 370 : bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4255 370 : ? data.clone_inbranch : data.clone);
4256 : else
4257 6096 : for (struct cgraph_node *n = node->simd_clones; n != NULL;
4258 5024 : n = n->simdclone->next_clone)
4259 : {
4260 5024 : unsigned int this_badness = 0;
4261 5024 : unsigned int num_calls;
4262 : /* The number of arguments in the call and the number of parameters in
4263 : the simdclone should match. However, when the simdclone is
4264 : 'inbranch', it could have one more parameter than nargs when using
4265 : an inbranch simdclone to call a non-inbranch call, either in a
4266 : non-masked loop using a all true constant mask, or inside a masked
4267 : loop using it's mask. */
4268 5024 : size_t simd_nargs = n->simdclone->nargs;
4269 5024 : if (!masked_call_offset && n->simdclone->inbranch)
4270 2259 : simd_nargs--;
4271 5024 : if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
4272 : &num_calls)
4273 1960 : || (!n->simdclone->inbranch && (masked_call_offset > 0))
4274 1776 : || (nargs != simd_nargs))
4275 3248 : continue;
4276 1776 : if (num_calls != 1)
4277 1162 : this_badness += floor_log2 (num_calls) * 4096;
4278 1776 : if (n->simdclone->inbranch)
4279 741 : this_badness += 8192;
4280 :
4281 : /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
4282 : mode, which for targets that use it will determine what ISA we can
4283 : vectorize this code with. */
4284 1776 : machine_mode vector_mode = vinfo->vector_mode;
4285 1776 : if (vectype)
4286 1776 : vector_mode = TYPE_MODE (vectype);
4287 1776 : int target_badness = targetm.simd_clone.usable (n, vector_mode);
4288 1776 : if (target_badness < 0)
4289 380 : continue;
4290 1396 : this_badness += target_badness * 512;
4291 4220 : for (i = 0; i < nargs; i++)
4292 : {
4293 3072 : switch (n->simdclone->args[i].arg_type)
4294 : {
4295 2142 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4296 2142 : if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
4297 : /* Vector mask arguments are not supported. */
4298 : i = -1;
4299 2134 : else if (!useless_type_conversion_p
4300 2134 : (n->simdclone->args[i].orig_type,
4301 2134 : TREE_TYPE (gimple_call_arg (stmt,
4302 : i + masked_call_offset))))
4303 : i = -1;
4304 2134 : else if (arginfo[i].dt == vect_constant_def
4305 2027 : || arginfo[i].dt == vect_external_def
4306 4097 : || arginfo[i].linear_step)
4307 399 : this_badness += 64;
4308 : break;
4309 310 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4310 310 : if ((arginfo[i].dt != vect_constant_def
4311 145 : && arginfo[i].dt != vect_external_def)
4312 410 : || SLP_TREE_LANES (slp_node) != 1)
4313 : i = -1;
4314 : break;
4315 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4316 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4317 324 : if (arginfo[i].dt == vect_constant_def
4318 324 : || arginfo[i].dt == vect_external_def
4319 324 : || (arginfo[i].linear_step
4320 324 : != n->simdclone->args[i].linear_step))
4321 : i = -1;
4322 : break;
4323 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4324 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4325 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4326 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4327 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4328 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4329 : /* FORNOW */
4330 : i = -1;
4331 : break;
4332 296 : case SIMD_CLONE_ARG_TYPE_MASK:
4333 296 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4334 264 : && n->simdclone->mask_mode != VOIDmode)
4335 : i = -1;
4336 : /* While we can create a traditional data vector from
4337 : an incoming integer mode mask we have no good way to
4338 : force generate an integer mode mask from a traditional
4339 : boolean vector input. */
4340 296 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4341 296 : && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4342 : i = -1;
4343 290 : else if (n->simdclone->mask_mode == VOIDmode
4344 : /* FORNOW we only have partial support for vector-type
4345 : masks that can't hold all of simdlen. */
4346 554 : && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
4347 429 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
4348 : /* Verify we can compute the mask argument. */
4349 111 : || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
4350 111 : arginfo[i].vectype)))
4351 : i = -1;
4352 125 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4353 : /* FORNOW we only have partial support for
4354 : integer-type masks that represent the same number
4355 : of lanes as the vectorized mask inputs. */
4356 151 : && maybe_ne (exact_div (n->simdclone->simdlen,
4357 : n->simdclone->args[i].linear_step),
4358 26 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4359 : i = -1;
4360 107 : else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4361 107 : && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4362 8 : this_badness += 2048;
4363 : break;
4364 : }
4365 18 : if (i == (size_t) -1)
4366 : break;
4367 2824 : if (n->simdclone->args[i].alignment > arginfo[i].align)
4368 : {
4369 : i = -1;
4370 : break;
4371 : }
4372 2824 : if (arginfo[i].align)
4373 110 : this_badness += (exact_log2 (arginfo[i].align)
4374 160 : - exact_log2 (n->simdclone->args[i].alignment));
4375 : }
4376 1396 : if (i == (size_t) -1)
4377 248 : continue;
4378 1148 : if (masked_call_offset == 0
4379 1041 : && n->simdclone->inbranch
4380 323 : && n->simdclone->nargs > nargs)
4381 : {
4382 323 : gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
4383 : SIMD_CLONE_ARG_TYPE_MASK);
4384 : /* Penalize using a masked SIMD clone in a non-masked loop, that is
4385 : not in a branch, as we'd have to construct an all-true mask. */
4386 323 : this_badness += 64;
4387 : }
4388 1148 : if (bestn == NULL || this_badness < badness)
4389 : {
4390 827 : bestn = n;
4391 827 : badness = this_badness;
4392 : }
4393 1148 : if (n->simdclone->inbranch
4394 430 : && (bestn_inbranch == NULL || this_badness < badness_inbranch))
4395 : {
4396 5024 : bestn_inbranch = n;
4397 5024 : badness_inbranch = this_badness;
4398 : }
4399 : }
4400 :
4401 1442 : if (bestn == NULL)
4402 : return false;
4403 :
4404 859 : fndecl = bestn->decl;
4405 859 : nunits = bestn->simdclone->simdlen;
4406 859 : int ncopies = vector_unroll_factor (vf * group_size, nunits);
4407 :
4408 : /* If the function isn't const, only allow it in simd loops where user
4409 : has asserted that at least nunits consecutive iterations can be
4410 : performed using SIMD instructions. */
4411 842 : if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
4412 1044 : && gimple_vuse (stmt))
4413 : return false;
4414 :
4415 : /* ncopies is the number of SIMD clone calls we create, since simdlen
4416 : is not necessarily matching nunits of the vector types used, track
4417 : that in ncopies_in. */
4418 859 : int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
4419 :
4420 : /* Sanity check: make sure that at least one copy of the vectorized stmt
4421 : needs to be generated. */
4422 859 : gcc_assert (ncopies >= 1);
4423 :
4424 859 : if (cost_vec) /* transformation not required. */
4425 : {
4426 1614 : for (unsigned i = 0; i < nargs; ++i)
4427 1125 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
4428 : {
4429 0 : if (dump_enabled_p ())
4430 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4431 : "incompatible vector types for invariants\n");
4432 : return false;
4433 : }
4434 :
4435 489 : if (!bestn_inbranch && loop_vinfo)
4436 : {
4437 270 : if (dump_enabled_p ()
4438 270 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
4439 193 : dump_printf_loc (MSG_NOTE, vect_location,
4440 : "can't use a fully-masked loop because no"
4441 : " masked simd clone was available.\n");
4442 270 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4443 : }
4444 :
4445 : /* When the original call is pure or const but the SIMD ABI dictates
4446 : an aggregate return we will have to use a virtual definition and
4447 : in a loop eventually even need to add a virtual PHI. That's
4448 : not straight-forward so allow to fix this up via renaming. */
4449 489 : if (gimple_call_lhs (stmt)
4450 483 : && !gimple_vdef (stmt)
4451 868 : && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
4452 33 : vinfo->any_known_not_updated_vssa = true;
4453 : /* ??? For SLP code-gen we end up inserting after the last
4454 : vector argument def rather than at the original call position
4455 : so automagic virtual operand updating doesn't work. */
4456 978 : if (gimple_vuse (stmt))
4457 147 : vinfo->any_known_not_updated_vssa = true;
4458 :
4459 489 : data.clone = bestn;
4460 489 : data.clone_inbranch = bestn_inbranch;
4461 :
4462 489 : simd_clone_info.safe_push (NULL_TREE);
4463 1763 : for (i = 0;
4464 2690 : i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
4465 : {
4466 1274 : if (loop_vinfo
4467 1250 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
4468 458 : && (bestn_inbranch->simdclone->args[i].arg_type
4469 : == SIMD_CLONE_ARG_TYPE_MASK))
4470 : {
4471 162 : if (masked_call_offset)
4472 : /* When there is an explicit mask we require the
4473 : number of elements to match up. */
4474 49 : vect_record_loop_mask (loop_vinfo,
4475 : &LOOP_VINFO_MASKS (loop_vinfo),
4476 : ncopies_in, vectype, NULL_TREE);
4477 : else
4478 : {
4479 : /* When there is no explicit mask on the call we have
4480 : more relaxed requirements. */
4481 113 : tree masktype;
4482 113 : poly_uint64 callee_nelements;
4483 113 : if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
4484 : {
4485 12 : callee_nelements
4486 12 : = exact_div (bestn_inbranch->simdclone->simdlen,
4487 : bestn_inbranch->simdclone->args[i].linear_step);
4488 12 : masktype = get_related_vectype_for_scalar_type
4489 12 : (vinfo->vector_mode, TREE_TYPE (vectype),
4490 : callee_nelements);
4491 : }
4492 : else
4493 : {
4494 101 : masktype = bestn_inbranch->simdclone->args[i].vector_type;
4495 : /* The aarch64 port will add custom attributes to types
4496 : for SVE simdclones which make the types different. We
4497 : should use canonincal types for masks within the
4498 : vectorizer, hence we construct the related vectype
4499 : here. */
4500 101 : masktype
4501 : = build_truth_vector_type_for_mode
4502 101 : (TYPE_VECTOR_SUBPARTS (masktype),
4503 101 : TYPE_MODE (masktype));
4504 101 : callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
4505 : }
4506 113 : auto o = vector_unroll_factor (nunits, callee_nelements);
4507 113 : vect_record_loop_mask (loop_vinfo,
4508 : &LOOP_VINFO_MASKS (loop_vinfo),
4509 : ncopies * o, masktype, NULL_TREE);
4510 : }
4511 : }
4512 1112 : else if ((bestn->simdclone->args[i].arg_type
4513 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4514 1005 : || (bestn->simdclone->args[i].arg_type
4515 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
4516 994 : || (bestn_inbranch
4517 388 : && ((bestn_inbranch->simdclone->args[i].arg_type
4518 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4519 388 : || (bestn_inbranch->simdclone->args[i].arg_type
4520 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
4521 : {
4522 118 : simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
4523 118 : simd_clone_info.safe_push (arginfo[i].op);
4524 202 : tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
4525 202 : ? size_type_node : TREE_TYPE (arginfo[i].op));
4526 118 : tree ls = build_int_cst (lst, arginfo[i].linear_step);
4527 118 : simd_clone_info.safe_push (ls);
4528 118 : tree sll = (arginfo[i].simd_lane_linear
4529 118 : ? boolean_true_node : boolean_false_node);
4530 118 : simd_clone_info.safe_push (sll);
4531 : }
4532 : }
4533 :
4534 489 : SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
4535 489 : slp_node->data = new vect_simd_clone_data (std::move (_data));
4536 489 : DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4537 : /* ??? We're confused by calls w/o LHS. */
4538 489 : if (SLP_TREE_VECTYPE (slp_node))
4539 483 : vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
4540 489 : return true;
4541 : }
4542 :
4543 : /* Transform. */
4544 :
4545 370 : if (dump_enabled_p ())
4546 248 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
4547 :
4548 : /* Handle def. */
4549 370 : scalar_dest = gimple_call_lhs (stmt);
4550 370 : vec_dest = NULL_TREE;
4551 370 : rtype = NULL_TREE;
4552 370 : ratype = NULL_TREE;
4553 370 : if (scalar_dest)
4554 : {
4555 364 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
4556 364 : rtype = TREE_TYPE (TREE_TYPE (fndecl));
4557 364 : if (TREE_CODE (rtype) == ARRAY_TYPE)
4558 : {
4559 9 : ratype = rtype;
4560 9 : rtype = TREE_TYPE (ratype);
4561 : }
4562 : }
4563 :
4564 740 : auto_vec<vec<tree> > vec_oprnds;
4565 370 : auto_vec<unsigned> vec_oprnds_i;
4566 370 : vec_oprnds_i.safe_grow_cleared (nargs, true);
4567 370 : vec_oprnds.reserve_exact (nargs);
4568 370 : vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
4569 1217 : for (j = 0; j < ncopies; ++j)
4570 : {
4571 477 : poly_uint64 callee_nelements;
4572 477 : poly_uint64 caller_nelements;
4573 : /* Build argument list for the vectorized call. */
4574 477 : if (j == 0)
4575 370 : vargs.create (nargs);
4576 : else
4577 107 : vargs.truncate (0);
4578 :
4579 1604 : for (i = 0; i < nargs; i++)
4580 : {
4581 1127 : unsigned int k, l, m, o;
4582 1127 : tree atype;
4583 1127 : tree op = gimple_call_arg (stmt, i + masked_call_offset);
4584 1127 : switch (bestn->simdclone->args[i].arg_type)
4585 : {
4586 838 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4587 838 : atype = bestn->simdclone->args[i].vector_type;
4588 838 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4589 838 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4590 838 : o = vector_unroll_factor (nunits, callee_nelements);
4591 1906 : for (m = j * o; m < (j + 1) * o; m++)
4592 : {
4593 1068 : if (known_lt (callee_nelements, caller_nelements))
4594 : {
4595 516 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
4596 258 : if (!constant_multiple_p (caller_nelements,
4597 : callee_nelements, &k))
4598 0 : gcc_unreachable ();
4599 :
4600 258 : gcc_assert ((k & (k - 1)) == 0);
4601 258 : if (m == 0)
4602 : {
4603 57 : vec_oprnds_i[i] = 0;
4604 57 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4605 : }
4606 : else
4607 : {
4608 201 : vec_oprnd0 = arginfo[i].op;
4609 201 : if ((m & (k - 1)) == 0)
4610 72 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4611 : }
4612 258 : arginfo[i].op = vec_oprnd0;
4613 258 : vec_oprnd0
4614 258 : = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
4615 258 : bitsize_int (prec),
4616 258 : bitsize_int ((m & (k - 1)) * prec));
4617 258 : gassign *new_stmt
4618 258 : = gimple_build_assign (make_ssa_name (atype),
4619 : vec_oprnd0);
4620 258 : vect_finish_stmt_generation (vinfo, stmt_info,
4621 : new_stmt, gsi);
4622 258 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4623 : }
4624 : else
4625 : {
4626 810 : if (!constant_multiple_p (callee_nelements,
4627 : caller_nelements, &k))
4628 0 : gcc_unreachable ();
4629 810 : gcc_assert ((k & (k - 1)) == 0);
4630 810 : vec<constructor_elt, va_gc> *ctor_elts;
4631 810 : if (k != 1)
4632 12 : vec_alloc (ctor_elts, k);
4633 : else
4634 798 : ctor_elts = NULL;
4635 834 : for (l = 0; l < k; l++)
4636 : {
4637 822 : if (m == 0 && l == 0)
4638 : {
4639 474 : vec_oprnds_i[i] = 0;
4640 474 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4641 : }
4642 : else
4643 348 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4644 822 : arginfo[i].op = vec_oprnd0;
4645 822 : if (k == 1)
4646 : break;
4647 24 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
4648 : vec_oprnd0);
4649 : }
4650 810 : if (k == 1)
4651 798 : if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
4652 : atype))
4653 : {
4654 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
4655 : vec_oprnd0);
4656 0 : gassign *new_stmt
4657 0 : = gimple_build_assign (make_ssa_name (atype),
4658 : vec_oprnd0);
4659 0 : vect_finish_stmt_generation (vinfo, stmt_info,
4660 : new_stmt, gsi);
4661 0 : vargs.safe_push (gimple_get_lhs (new_stmt));
4662 : }
4663 : else
4664 798 : vargs.safe_push (vec_oprnd0);
4665 : else
4666 : {
4667 12 : vec_oprnd0 = build_constructor (atype, ctor_elts);
4668 12 : gassign *new_stmt
4669 12 : = gimple_build_assign (make_ssa_name (atype),
4670 : vec_oprnd0);
4671 12 : vect_finish_stmt_generation (vinfo, stmt_info,
4672 : new_stmt, gsi);
4673 12 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4674 : }
4675 : }
4676 : }
4677 : break;
4678 66 : case SIMD_CLONE_ARG_TYPE_MASK:
4679 66 : if (bestn->simdclone->mask_mode == VOIDmode)
4680 : {
4681 60 : atype = bestn->simdclone->args[i].vector_type;
4682 60 : tree elt_type = TREE_TYPE (atype);
4683 60 : tree one = fold_convert (elt_type, integer_one_node);
4684 60 : tree zero = fold_convert (elt_type, integer_zero_node);
4685 60 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4686 60 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4687 60 : o = vector_unroll_factor (nunits, callee_nelements);
4688 120 : for (m = j * o; m < (j + 1) * o; m++)
4689 : {
4690 60 : if (maybe_lt (callee_nelements, caller_nelements))
4691 : {
4692 : /* The mask type has fewer elements than simdlen. */
4693 :
4694 : /* FORNOW */
4695 0 : gcc_unreachable ();
4696 : }
4697 60 : else if (known_eq (callee_nelements, caller_nelements))
4698 : {
4699 : /* The SIMD clone function has the same number of
4700 : elements as the current function. */
4701 60 : if (m == 0)
4702 60 : vec_oprnds_i[i] = 0;
4703 60 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4704 60 : if (loop_vinfo
4705 60 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4706 : {
4707 0 : vec_loop_masks *loop_masks
4708 : = &LOOP_VINFO_MASKS (loop_vinfo);
4709 0 : tree loop_mask
4710 0 : = vect_get_loop_mask (loop_vinfo, gsi,
4711 : loop_masks, ncopies_in,
4712 0 : vectype, j);
4713 0 : vec_oprnd0
4714 0 : = prepare_vec_mask (loop_vinfo,
4715 0 : TREE_TYPE (loop_mask),
4716 : loop_mask, vec_oprnd0,
4717 : gsi);
4718 0 : loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
4719 : loop_mask });
4720 :
4721 : }
4722 60 : vec_oprnd0
4723 60 : = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
4724 : build_vector_from_val (atype, one),
4725 : build_vector_from_val (atype, zero));
4726 60 : gassign *new_stmt
4727 60 : = gimple_build_assign (make_ssa_name (atype),
4728 : vec_oprnd0);
4729 60 : vect_finish_stmt_generation (vinfo, stmt_info,
4730 : new_stmt, gsi);
4731 60 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4732 : }
4733 : else
4734 : {
4735 : /* The mask type has more elements than simdlen. */
4736 :
4737 : /* FORNOW */
4738 0 : gcc_unreachable ();
4739 : }
4740 : }
4741 : }
4742 6 : else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4743 : {
4744 6 : atype = bestn->simdclone->args[i].vector_type;
4745 6 : poly_uint64 atype_subparts
4746 6 : = exact_div (bestn->simdclone->simdlen,
4747 : bestn->simdclone->args[i].linear_step);
4748 6 : o = bestn->simdclone->args[i].linear_step;
4749 12 : for (m = j * o; m < (j + 1) * o; m++)
4750 : {
4751 6 : if (m == 0)
4752 6 : vec_oprnds_i[i] = 0;
4753 6 : if (maybe_lt (atype_subparts,
4754 6 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4755 : {
4756 : /* The mask argument has fewer elements than the
4757 : input vector. */
4758 : /* FORNOW */
4759 0 : gcc_unreachable ();
4760 : }
4761 6 : else if (known_eq (atype_subparts,
4762 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4763 : {
4764 6 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4765 6 : if (loop_vinfo
4766 6 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4767 : {
4768 1 : vec_loop_masks *loop_masks
4769 : = &LOOP_VINFO_MASKS (loop_vinfo);
4770 1 : tree loop_mask
4771 1 : = vect_get_loop_mask (loop_vinfo, gsi,
4772 : loop_masks, ncopies_in,
4773 : vectype, j);
4774 1 : vec_oprnd0
4775 1 : = prepare_vec_mask (loop_vinfo,
4776 1 : TREE_TYPE (loop_mask),
4777 : loop_mask, vec_oprnd0,
4778 : gsi);
4779 : }
4780 : /* The vector mask argument matches the input
4781 : in the number of lanes, but not necessarily
4782 : in the mode. */
4783 6 : tree st = lang_hooks.types.type_for_mode
4784 6 : (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
4785 6 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
4786 : vec_oprnd0);
4787 6 : gassign *new_stmt
4788 6 : = gimple_build_assign (make_ssa_name (st),
4789 : vec_oprnd0);
4790 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4791 : new_stmt, gsi);
4792 6 : if (!types_compatible_p (atype, st))
4793 : {
4794 6 : new_stmt
4795 6 : = gimple_build_assign (make_ssa_name (atype),
4796 : NOP_EXPR,
4797 : gimple_assign_lhs
4798 : (new_stmt));
4799 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4800 : new_stmt, gsi);
4801 : }
4802 6 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4803 : }
4804 : else
4805 : {
4806 : /* The mask argument has more elements than the
4807 : input vector. */
4808 : /* FORNOW */
4809 0 : gcc_unreachable ();
4810 : }
4811 : }
4812 : }
4813 : else
4814 0 : gcc_unreachable ();
4815 : break;
4816 102 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4817 102 : vargs.safe_push (op);
4818 102 : break;
4819 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4820 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4821 121 : if (j == 0)
4822 : {
4823 118 : gimple_seq stmts;
4824 118 : arginfo[i].op
4825 118 : = force_gimple_operand (unshare_expr (arginfo[i].op),
4826 : &stmts, true, NULL_TREE);
4827 118 : if (stmts != NULL)
4828 : {
4829 0 : basic_block new_bb;
4830 0 : edge pe = loop_preheader_edge (loop);
4831 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4832 0 : gcc_assert (!new_bb);
4833 : }
4834 118 : if (arginfo[i].simd_lane_linear)
4835 : {
4836 6 : vargs.safe_push (arginfo[i].op);
4837 6 : break;
4838 : }
4839 112 : tree phi_res = copy_ssa_name (op);
4840 112 : gphi *new_phi = create_phi_node (phi_res, loop->header);
4841 112 : add_phi_arg (new_phi, arginfo[i].op,
4842 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
4843 112 : enum tree_code code
4844 196 : = POINTER_TYPE_P (TREE_TYPE (op))
4845 112 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4846 196 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4847 196 : ? sizetype : TREE_TYPE (op);
4848 112 : poly_widest_int cst
4849 112 : = wi::mul (bestn->simdclone->args[i].linear_step,
4850 112 : ncopies * nunits);
4851 112 : tree tcst = wide_int_to_tree (type, cst);
4852 112 : tree phi_arg = copy_ssa_name (op);
4853 112 : gassign *new_stmt
4854 112 : = gimple_build_assign (phi_arg, code, phi_res, tcst);
4855 112 : gimple_stmt_iterator si = gsi_after_labels (loop->header);
4856 112 : gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4857 112 : add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4858 : UNKNOWN_LOCATION);
4859 112 : arginfo[i].op = phi_res;
4860 112 : vargs.safe_push (phi_res);
4861 112 : }
4862 : else
4863 : {
4864 3 : enum tree_code code
4865 6 : = POINTER_TYPE_P (TREE_TYPE (op))
4866 3 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4867 6 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4868 6 : ? sizetype : TREE_TYPE (op);
4869 3 : poly_widest_int cst
4870 3 : = wi::mul (bestn->simdclone->args[i].linear_step,
4871 3 : j * nunits);
4872 3 : tree tcst = wide_int_to_tree (type, cst);
4873 3 : new_temp = make_ssa_name (TREE_TYPE (op));
4874 3 : gassign *new_stmt
4875 6 : = gimple_build_assign (new_temp, code,
4876 3 : arginfo[i].op, tcst);
4877 3 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4878 3 : vargs.safe_push (new_temp);
4879 3 : }
4880 : break;
4881 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4882 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4883 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4884 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4885 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4886 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4887 0 : default:
4888 0 : gcc_unreachable ();
4889 : }
4890 : }
4891 :
4892 477 : if (masked_call_offset == 0
4893 411 : && bestn->simdclone->inbranch
4894 25 : && bestn->simdclone->nargs > nargs)
4895 : {
4896 25 : unsigned long m, o;
4897 25 : size_t mask_i = bestn->simdclone->nargs - 1;
4898 25 : tree mask;
4899 25 : gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
4900 : SIMD_CLONE_ARG_TYPE_MASK);
4901 :
4902 25 : tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
4903 25 : tree mask_vectype;
4904 25 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4905 : {
4906 2 : callee_nelements = exact_div (bestn->simdclone->simdlen,
4907 : bestn->simdclone->args[i].linear_step);
4908 2 : mask_vectype = get_related_vectype_for_scalar_type
4909 2 : (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
4910 : }
4911 : else
4912 : {
4913 23 : mask_vectype = mask_argtype;
4914 23 : callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
4915 : }
4916 25 : o = vector_unroll_factor (nunits, callee_nelements);
4917 50 : for (m = j * o; m < (j + 1) * o; m++)
4918 : {
4919 25 : if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4920 : {
4921 1 : vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
4922 1 : mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
4923 : ncopies * o, mask_vectype, m);
4924 : }
4925 24 : else if (bestn->simdclone->mask_mode == VOIDmode)
4926 : {
4927 : /* All lanes are active: pass what the VEC_COND_EXPR below
4928 : would select. An all-ones floating-point MASK_ARGTYPE
4929 : is not a valid condition. */
4930 23 : vargs.safe_push (build_one_cst (mask_argtype));
4931 23 : continue;
4932 : }
4933 : else
4934 1 : mask = vect_build_all_ones_mask (vinfo, stmt_info,
4935 : mask_argtype);
4936 :
4937 2 : gassign *new_stmt;
4938 2 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4939 : {
4940 : /* This means we are dealing with integer mask modes.
4941 : First convert to an integer type with the same size as
4942 : the current vector type. */
4943 2 : unsigned HOST_WIDE_INT intermediate_size
4944 2 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
4945 2 : tree mid_int_type =
4946 2 : build_nonstandard_integer_type (intermediate_size, 1);
4947 2 : mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
4948 2 : new_stmt
4949 2 : = gimple_build_assign (make_ssa_name (mid_int_type),
4950 : mask);
4951 2 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4952 : /* Then zero-extend to the mask mode. */
4953 2 : mask = fold_build1 (NOP_EXPR, mask_argtype,
4954 : gimple_get_lhs (new_stmt));
4955 : }
4956 0 : else if (bestn->simdclone->mask_mode == VOIDmode)
4957 0 : mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
4958 : build_one_cst (mask_argtype),
4959 : build_zero_cst (mask_argtype));
4960 : else
4961 0 : gcc_unreachable ();
4962 :
4963 2 : new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
4964 : mask);
4965 2 : vect_finish_stmt_generation (vinfo, stmt_info,
4966 : new_stmt, gsi);
4967 2 : mask = gimple_assign_lhs (new_stmt);
4968 2 : vargs.safe_push (mask);
4969 : }
4970 : }
4971 :
4972 477 : gcall *new_call = gimple_build_call_vec (fndecl, vargs);
4973 477 : if (vec_dest)
4974 : {
4975 471 : gcc_assert (ratype
4976 : || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
4977 471 : if (ratype)
4978 15 : new_temp = create_tmp_var (ratype);
4979 456 : else if (useless_type_conversion_p (vectype, rtype))
4980 436 : new_temp = make_ssa_name (vec_dest, new_call);
4981 : else
4982 20 : new_temp = make_ssa_name (rtype, new_call);
4983 471 : gimple_call_set_lhs (new_call, new_temp);
4984 : }
4985 477 : vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
4986 477 : gimple *new_stmt = new_call;
4987 :
4988 477 : if (vec_dest)
4989 : {
4990 471 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
4991 : {
4992 19 : unsigned int k, l;
4993 38 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
4994 38 : poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
4995 19 : k = vector_unroll_factor (nunits,
4996 : TYPE_VECTOR_SUBPARTS (vectype));
4997 19 : gcc_assert ((k & (k - 1)) == 0);
4998 69 : for (l = 0; l < k; l++)
4999 : {
5000 50 : tree t;
5001 50 : if (ratype)
5002 : {
5003 42 : t = build_fold_addr_expr (new_temp);
5004 42 : t = build2 (MEM_REF, vectype, t,
5005 42 : build_int_cst (TREE_TYPE (t), l * bytes));
5006 : }
5007 : else
5008 8 : t = build3 (BIT_FIELD_REF, vectype, new_temp,
5009 8 : bitsize_int (prec), bitsize_int (l * prec));
5010 50 : new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
5011 50 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5012 :
5013 50 : SLP_TREE_VEC_DEFS (slp_node)
5014 50 : .quick_push (gimple_assign_lhs (new_stmt));
5015 : }
5016 :
5017 19 : if (ratype)
5018 15 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5019 19 : continue;
5020 19 : }
5021 452 : else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
5022 : {
5023 16 : unsigned int k;
5024 16 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
5025 16 : TYPE_VECTOR_SUBPARTS (rtype), &k))
5026 0 : gcc_unreachable ();
5027 16 : gcc_assert ((k & (k - 1)) == 0);
5028 16 : if ((j & (k - 1)) == 0)
5029 8 : vec_alloc (ret_ctor_elts, k);
5030 16 : if (ratype)
5031 : {
5032 0 : unsigned int m, o;
5033 0 : o = vector_unroll_factor (nunits,
5034 : TYPE_VECTOR_SUBPARTS (rtype));
5035 0 : for (m = 0; m < o; m++)
5036 : {
5037 0 : tree tem = build4 (ARRAY_REF, rtype, new_temp,
5038 0 : size_int (m), NULL_TREE, NULL_TREE);
5039 0 : new_stmt = gimple_build_assign (make_ssa_name (rtype),
5040 : tem);
5041 0 : vect_finish_stmt_generation (vinfo, stmt_info,
5042 : new_stmt, gsi);
5043 0 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
5044 : gimple_assign_lhs (new_stmt));
5045 : }
5046 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5047 : }
5048 : else
5049 16 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
5050 16 : if ((j & (k - 1)) != k - 1)
5051 8 : continue;
5052 8 : vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
5053 8 : new_stmt
5054 8 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5055 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5056 :
5057 8 : SLP_TREE_VEC_DEFS (slp_node)
5058 8 : .quick_push (gimple_assign_lhs (new_stmt));
5059 8 : continue;
5060 8 : }
5061 436 : else if (ratype)
5062 : {
5063 0 : tree t = build_fold_addr_expr (new_temp);
5064 0 : t = build2 (MEM_REF, vectype, t,
5065 0 : build_int_cst (TREE_TYPE (t), 0));
5066 0 : new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
5067 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5068 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5069 : }
5070 436 : else if (!useless_type_conversion_p (vectype, rtype))
5071 : {
5072 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
5073 0 : new_stmt
5074 0 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5075 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5076 : }
5077 : }
5078 :
5079 442 : if (gimple_get_lhs (new_stmt))
5080 436 : SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
5081 : }
5082 :
5083 1187 : for (i = 0; i < nargs; ++i)
5084 : {
5085 817 : vec<tree> oprndsi = vec_oprnds[i];
5086 817 : oprndsi.release ();
5087 : }
5088 370 : vargs.release ();
5089 :
5090 : /* Mark the clone as no longer being a candidate for GC. */
5091 370 : bestn->gc_candidate = false;
5092 :
5093 370 : return true;
5094 1442 : }
5095 :
5096 :
5097 : /* Function vect_gen_widened_results_half
5098 :
5099 : Create a vector stmt whose code, type, number of arguments, and result
5100 : variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5101 : VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5102 : In the case that CODE is a CALL_EXPR, this means that a call to DECL
5103 : needs to be created (DECL is a function-decl of a target-builtin).
5104 : STMT_INFO is the original scalar stmt that we are vectorizing. */
5105 :
5106 : static gimple *
5107 31781 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
5108 : tree vec_oprnd0, tree vec_oprnd1, int op_type,
5109 : tree vec_dest, gimple_stmt_iterator *gsi,
5110 : stmt_vec_info stmt_info)
5111 : {
5112 31781 : gimple *new_stmt;
5113 31781 : tree new_temp;
5114 :
5115 : /* Generate half of the widened result: */
5116 31781 : if (op_type != binary_op)
5117 30645 : vec_oprnd1 = NULL;
5118 31781 : new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
5119 31781 : new_temp = make_ssa_name (vec_dest, new_stmt);
5120 31781 : gimple_set_lhs (new_stmt, new_temp);
5121 31781 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5122 :
5123 31781 : return new_stmt;
5124 : }
5125 :
5126 :
5127 : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5128 : For multi-step conversions store the resulting vectors and call the function
5129 : recursively. When NARROW_SRC_P is true, there's still a conversion after
5130 : narrowing, don't store the vectors in the SLP_NODE or in vector info of
5131 : the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5132 :
5133 : static void
5134 12853 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
5135 : int multi_step_cvt,
5136 : stmt_vec_info stmt_info,
5137 : vec<tree> &vec_dsts,
5138 : gimple_stmt_iterator *gsi,
5139 : slp_tree slp_node, code_helper code,
5140 : bool narrow_src_p)
5141 : {
5142 12853 : unsigned int i;
5143 12853 : tree vop0, vop1, new_tmp, vec_dest;
5144 :
5145 12853 : vec_dest = vec_dsts.pop ();
5146 :
5147 30239 : for (i = 0; i < vec_oprnds->length (); i += 2)
5148 : {
5149 : /* Create demotion operation. */
5150 17386 : vop0 = (*vec_oprnds)[i];
5151 17386 : vop1 = (*vec_oprnds)[i + 1];
5152 17386 : gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
5153 17386 : new_tmp = make_ssa_name (vec_dest, new_stmt);
5154 17386 : gimple_set_lhs (new_stmt, new_tmp);
5155 17386 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5156 17386 : if (multi_step_cvt || narrow_src_p)
5157 : /* Store the resulting vector for next recursive call,
5158 : or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5159 6948 : (*vec_oprnds)[i/2] = new_tmp;
5160 : else
5161 : {
5162 : /* This is the last step of the conversion sequence. Store the
5163 : vectors in SLP_NODE. */
5164 10438 : slp_node->push_vec_def (new_stmt);
5165 : }
5166 : }
5167 :
5168 : /* For multi-step demotion operations we first generate demotion operations
5169 : from the source type to the intermediate types, and then combine the
5170 : results (stored in VEC_OPRNDS) in demotion operation to the destination
5171 : type. */
5172 12853 : if (multi_step_cvt)
5173 : {
5174 : /* At each level of recursion we have half of the operands we had at the
5175 : previous level. */
5176 3088 : vec_oprnds->truncate ((i+1)/2);
5177 3088 : vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
5178 : multi_step_cvt - 1,
5179 : stmt_info, vec_dsts, gsi,
5180 3088 : slp_node, VEC_PACK_TRUNC_EXPR,
5181 : narrow_src_p);
5182 : }
5183 :
5184 12853 : vec_dsts.quick_push (vec_dest);
5185 12853 : }
5186 :
5187 :
5188 : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5189 : and VEC_OPRNDS1, for a binary operation associated with scalar statement
5190 : STMT_INFO. For multi-step conversions store the resulting vectors and
5191 : call the function recursively. When NUM_VECTORS is not -1U then only
5192 : NUM_VECTORS will be produced. */
5193 :
5194 : static void
5195 11572 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
5196 : vec<tree> *vec_oprnds0,
5197 : vec<tree> *vec_oprnds1,
5198 : stmt_vec_info stmt_info, tree vec_dest,
5199 : gimple_stmt_iterator *gsi,
5200 : code_helper ch1,
5201 : code_helper ch2, int op_type,
5202 : unsigned num_vectors)
5203 : {
5204 11572 : int i;
5205 11572 : tree vop0, vop1, new_tmp1, new_tmp2;
5206 11572 : gimple *new_stmt1, *new_stmt2;
5207 11572 : vec<tree> vec_tmp = vNULL;
5208 :
5209 11572 : vec_tmp.create (num_vectors != -1U
5210 1825 : ? num_vectors : 2 * vec_oprnds0->length ());
5211 27468 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5212 : {
5213 15896 : if (op_type == binary_op)
5214 568 : vop1 = (*vec_oprnds1)[i];
5215 : else
5216 : vop1 = NULL_TREE;
5217 :
5218 : /* Generate the two halves of promotion operation. */
5219 15896 : new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
5220 : op_type, vec_dest, gsi,
5221 : stmt_info);
5222 15896 : new_tmp1 = gimple_get_lhs (new_stmt1);
5223 15896 : vec_tmp.quick_push (new_tmp1);
5224 :
5225 15896 : if (vec_tmp.space (1))
5226 : {
5227 15885 : new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
5228 : op_type, vec_dest, gsi,
5229 : stmt_info);
5230 15885 : new_tmp2 = gimple_get_lhs (new_stmt2);
5231 15885 : vec_tmp.quick_push (new_tmp2);
5232 : }
5233 :
5234 15896 : if (!vec_tmp.space (1))
5235 : break;
5236 : }
5237 :
5238 11572 : vec_oprnds0->release ();
5239 11572 : *vec_oprnds0 = vec_tmp;
5240 11572 : }
5241 :
5242 : /* Create vectorized promotion stmts for widening stmts using only half the
5243 : potential vector size for input. */
5244 : static void
5245 44 : vect_create_half_widening_stmts (vec_info *vinfo,
5246 : vec<tree> *vec_oprnds0,
5247 : vec<tree> *vec_oprnds1,
5248 : stmt_vec_info stmt_info, tree vec_dest,
5249 : gimple_stmt_iterator *gsi,
5250 : code_helper code1,
5251 : int op_type)
5252 : {
5253 44 : int i;
5254 44 : tree vop0, vop1;
5255 44 : gimple *new_stmt1;
5256 44 : gimple *new_stmt2;
5257 44 : gimple *new_stmt3;
5258 44 : vec<tree> vec_tmp = vNULL;
5259 :
5260 44 : vec_tmp.create (vec_oprnds0->length ());
5261 132 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5262 : {
5263 44 : tree new_tmp1, new_tmp2, new_tmp3, out_type;
5264 :
5265 44 : gcc_assert (op_type == binary_op);
5266 44 : vop1 = (*vec_oprnds1)[i];
5267 :
5268 : /* Widen the first vector input. */
5269 44 : out_type = TREE_TYPE (vec_dest);
5270 44 : new_tmp1 = make_ssa_name (out_type);
5271 44 : new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
5272 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
5273 44 : if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
5274 : {
5275 : /* Widen the second vector input. */
5276 44 : new_tmp2 = make_ssa_name (out_type);
5277 44 : new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
5278 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
5279 : /* Perform the operation. With both vector inputs widened. */
5280 44 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
5281 : }
5282 : else
5283 : {
5284 : /* Perform the operation. With the single vector input widened. */
5285 0 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
5286 : }
5287 :
5288 44 : new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
5289 44 : gimple_assign_set_lhs (new_stmt3, new_tmp3);
5290 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
5291 :
5292 : /* Store the results for the next step. */
5293 44 : vec_tmp.quick_push (new_tmp3);
5294 : }
5295 :
5296 44 : vec_oprnds0->release ();
5297 44 : *vec_oprnds0 = vec_tmp;
5298 44 : }
5299 :
5300 :
5301 : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5302 : If COST_VEC is passed, calculate costs but don't change anything,
5303 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5304 : it, and insert it at GSI.
5305 : Return true if STMT_INFO is vectorizable in this way. */
5306 :
5307 : static bool
5308 2676861 : vectorizable_conversion (vec_info *vinfo,
5309 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5310 : slp_tree slp_node,
5311 : stmt_vector_for_cost *cost_vec)
5312 : {
5313 2676861 : tree vec_dest, cvt_op = NULL_TREE;
5314 2676861 : tree scalar_dest;
5315 2676861 : tree_code tc1;
5316 2676861 : code_helper code, code1, code2;
5317 2676861 : code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
5318 2676861 : tree new_temp;
5319 2676861 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5320 2676861 : poly_uint64 nunits_in;
5321 2676861 : poly_uint64 nunits_out;
5322 2676861 : tree vectype_out, vectype_in;
5323 2676861 : int i;
5324 2676861 : tree lhs_type, rhs_type;
5325 : /* For conversions between floating point and integer, there're 2 NARROW
5326 : cases. NARROW_SRC is for FLOAT_EXPR, means
5327 : integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5328 : This is safe when the range of the source integer can fit into the lower
5329 : precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5330 : floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5331 : For other conversions, when there's narrowing, NARROW_DST is used as
5332 : default. */
5333 2676861 : enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
5334 2676861 : vec<tree> vec_oprnds0 = vNULL;
5335 2676861 : vec<tree> vec_oprnds1 = vNULL;
5336 2676861 : tree vop0;
5337 2676861 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5338 2676861 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5339 2676861 : int multi_step_cvt = 0;
5340 2676861 : vec<tree> interm_types = vNULL;
5341 2676861 : tree intermediate_type, cvt_type = NULL_TREE;
5342 2676861 : int op_type;
5343 2676861 : unsigned short fltsz;
5344 :
5345 : /* Is STMT a vectorizable conversion? */
5346 :
5347 2676861 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5348 : return false;
5349 :
5350 2676861 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5351 247311 : && cost_vec)
5352 : return false;
5353 :
5354 2429550 : gimple* stmt = stmt_info->stmt;
5355 2429550 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
5356 : return false;
5357 :
5358 2367830 : if (gimple_get_lhs (stmt) == NULL_TREE
5359 2367830 : || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5360 : return false;
5361 :
5362 1532684 : if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5363 : return false;
5364 :
5365 1532684 : if (is_gimple_assign (stmt))
5366 : {
5367 1519838 : code = gimple_assign_rhs_code (stmt);
5368 1519838 : op_type = TREE_CODE_LENGTH ((tree_code) code);
5369 : }
5370 12846 : else if (gimple_call_internal_p (stmt))
5371 : {
5372 7657 : code = gimple_call_internal_fn (stmt);
5373 7657 : op_type = gimple_call_num_args (stmt);
5374 : }
5375 : else
5376 : return false;
5377 :
5378 1527495 : bool widen_arith = (code == WIDEN_MULT_EXPR
5379 1524899 : || code == WIDEN_LSHIFT_EXPR
5380 3052394 : || widening_fn_p (code));
5381 :
5382 1524899 : if (!widen_arith
5383 1524899 : && !CONVERT_EXPR_CODE_P (code)
5384 : && code != FIX_TRUNC_EXPR
5385 : && code != FLOAT_EXPR)
5386 : return false;
5387 :
5388 : /* Check types of lhs and rhs. */
5389 200733 : scalar_dest = gimple_get_lhs (stmt);
5390 200733 : lhs_type = TREE_TYPE (scalar_dest);
5391 200733 : vectype_out = SLP_TREE_VECTYPE (slp_node);
5392 :
5393 : /* Check the operands of the operation. */
5394 200733 : slp_tree slp_op0, slp_op1 = NULL;
5395 200733 : if (!vect_is_simple_use (vinfo, slp_node,
5396 : 0, &slp_op0, &dt[0], &vectype_in))
5397 : {
5398 0 : if (dump_enabled_p ())
5399 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5400 : "use not simple.\n");
5401 : return false;
5402 : }
5403 :
5404 200733 : rhs_type = TREE_TYPE (gimple_arg (stmt, 0));
5405 200733 : if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
5406 381574 : && !((INTEGRAL_TYPE_P (lhs_type)
5407 166100 : && INTEGRAL_TYPE_P (rhs_type))
5408 : || (SCALAR_FLOAT_TYPE_P (lhs_type)
5409 9874 : && SCALAR_FLOAT_TYPE_P (rhs_type))))
5410 : return false;
5411 :
5412 195866 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5413 175595 : && INTEGRAL_TYPE_P (lhs_type)
5414 339691 : && !type_has_mode_precision_p (lhs_type))
5415 : {
5416 571 : if (dump_enabled_p ())
5417 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5418 : "type conversion to bit-precision unsupported\n");
5419 : return false;
5420 : }
5421 :
5422 : /* _BitInt values are not sign-/zero-extended to mode precision. */
5423 195295 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5424 175024 : && TREE_CODE (rhs_type) == BITINT_TYPE
5425 195484 : && !type_has_mode_precision_p (rhs_type))
5426 : {
5427 32 : if (dump_enabled_p ())
5428 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5429 : "type conversion from _BitInt unsupported\n");
5430 : return false;
5431 : }
5432 :
5433 195263 : if (op_type == binary_op)
5434 : {
5435 2596 : gcc_assert (code == WIDEN_MULT_EXPR
5436 : || code == WIDEN_LSHIFT_EXPR
5437 : || widening_fn_p (code));
5438 :
5439 2596 : tree vectype1_in;
5440 2596 : if (!vect_is_simple_use (vinfo, slp_node, 1,
5441 : &slp_op1, &dt[1], &vectype1_in))
5442 : {
5443 0 : if (dump_enabled_p ())
5444 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5445 : "use not simple.\n");
5446 0 : return false;
5447 : }
5448 : /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5449 : OP1. */
5450 2596 : if (!vectype_in)
5451 165 : vectype_in = vectype1_in;
5452 : }
5453 :
5454 : /* If slp_op0 is an external or constant def, infer the vector type
5455 : from the scalar type. */
5456 195263 : if (!cost_vec)
5457 24025 : gcc_assert (vectype_in);
5458 195263 : if (!vectype_in)
5459 25193 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
5460 195263 : if (!vectype_in)
5461 : {
5462 276 : if (dump_enabled_p ())
5463 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5464 : "no vectype for scalar type %T\n", rhs_type);
5465 :
5466 : return false;
5467 : }
5468 :
5469 389974 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
5470 194987 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
5471 : {
5472 309 : if (dump_enabled_p ())
5473 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5474 : "can't convert between boolean and non "
5475 : "boolean vectors %T\n", rhs_type);
5476 :
5477 : return false;
5478 : }
5479 :
5480 194678 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
5481 194678 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5482 194678 : if (known_eq (nunits_out, nunits_in))
5483 93370 : if (widen_arith)
5484 : modifier = WIDEN;
5485 : else
5486 194678 : modifier = NONE;
5487 101308 : else if (multiple_p (nunits_out, nunits_in))
5488 : modifier = NARROW_DST;
5489 : else
5490 : {
5491 56483 : gcc_checking_assert (multiple_p (nunits_in, nunits_out));
5492 : modifier = WIDEN;
5493 : }
5494 :
5495 194678 : bool found_mode = false;
5496 194678 : scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
5497 194678 : scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
5498 194678 : opt_scalar_mode rhs_mode_iter;
5499 194678 : auto_vec<std::pair<tree, tree_code>, 2> converts;
5500 194678 : bool evenodd_ok = false;
5501 :
5502 : /* Supportable by target? */
5503 194678 : switch (modifier)
5504 : {
5505 93011 : case NONE:
5506 93011 : if (code != FIX_TRUNC_EXPR
5507 : && code != FLOAT_EXPR
5508 : && !CONVERT_EXPR_CODE_P (code))
5509 : return false;
5510 93011 : gcc_assert (code.is_tree_code ());
5511 93011 : if (supportable_indirect_convert_operation (code,
5512 : vectype_out, vectype_in,
5513 : converts, slp_op0))
5514 : {
5515 86098 : gcc_assert (converts.length () <= 2);
5516 86098 : if (converts.length () == 1)
5517 : {
5518 86024 : code1 = converts[0].second;
5519 86024 : if (CONVERT_EXPR_CODE_P (code)
5520 161587 : && tree_nop_conversion_p (TREE_TYPE (vectype_out),
5521 75563 : TREE_TYPE (vectype_in)))
5522 : /* NOP conversions are handled by vectorizable_assignment. */
5523 : return false;
5524 : }
5525 : else
5526 : {
5527 74 : cvt_type = NULL_TREE;
5528 74 : multi_step_cvt = converts.length () - 1;
5529 74 : codecvt1 = converts[0].second;
5530 74 : code1 = converts[1].second;
5531 74 : interm_types.safe_push (converts[0].first);
5532 : }
5533 : break;
5534 : }
5535 :
5536 : /* FALLTHRU */
5537 6913 : unsupported:
5538 19481 : if (dump_enabled_p ())
5539 570 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5540 : "conversion not supported by target.\n");
5541 : return false;
5542 :
5543 56842 : case WIDEN:
5544 56842 : if (known_eq (nunits_in, nunits_out))
5545 : {
5546 718 : if (!(code.is_tree_code ()
5547 359 : && supportable_half_widening_operation ((tree_code) code,
5548 : vectype_out, vectype_in,
5549 : &tc1)))
5550 119 : goto unsupported;
5551 240 : code1 = tc1;
5552 240 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5553 : break;
5554 : }
5555 : /* Elements in a vector can only be reordered if used in a reduction
5556 : operation only. */
5557 56483 : if (code == WIDEN_MULT_EXPR
5558 2237 : && loop_vinfo
5559 2096 : && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
5560 : /* For a SLP reduction we cannot swizzle lanes, detecting a
5561 : reduction chain isn't possible here. */
5562 58557 : && SLP_TREE_LANES (slp_node) == 1)
5563 : {
5564 : /* ??? There is no way to look for SLP uses, so work on
5565 : the stmt and what the stmt-based cycle detection gives us. */
5566 3944 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5567 1972 : stmt_vec_info use_stmt_info
5568 1972 : = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
5569 1972 : if (use_stmt_info
5570 1827 : && STMT_VINFO_REDUC_DEF (use_stmt_info))
5571 56483 : evenodd_ok = true;
5572 : }
5573 56483 : if (supportable_widening_operation (code, vectype_out, vectype_in,
5574 : evenodd_ok, &code1,
5575 : &code2, &multi_step_cvt,
5576 : &interm_types))
5577 : {
5578 : /* Binary widening operation can only be supported directly by the
5579 : architecture. */
5580 50089 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5581 : break;
5582 : }
5583 :
5584 6394 : if (code != FLOAT_EXPR
5585 6760 : || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
5586 6211 : goto unsupported;
5587 :
5588 183 : fltsz = GET_MODE_SIZE (lhs_mode);
5589 270 : FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
5590 : {
5591 270 : rhs_mode = rhs_mode_iter.require ();
5592 540 : if (GET_MODE_SIZE (rhs_mode) > fltsz)
5593 : break;
5594 :
5595 270 : cvt_type
5596 270 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5597 270 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5598 270 : if (cvt_type == NULL_TREE)
5599 0 : goto unsupported;
5600 :
5601 540 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5602 : {
5603 78 : gcc_assert (code.is_tree_code ());
5604 78 : if (!supportable_convert_operation ((tree_code) code, vectype_out,
5605 : cvt_type))
5606 22 : goto unsupported;
5607 56 : codecvt1 = code;
5608 : }
5609 192 : else if (!supportable_widening_operation (code, vectype_out,
5610 : cvt_type, evenodd_ok,
5611 : &codecvt1,
5612 : &codecvt2, &multi_step_cvt,
5613 : &interm_types))
5614 87 : continue;
5615 : else
5616 105 : gcc_assert (multi_step_cvt == 0);
5617 :
5618 161 : if (supportable_widening_operation (NOP_EXPR, cvt_type,
5619 : vectype_in, evenodd_ok, &code1,
5620 : &code2, &multi_step_cvt,
5621 : &interm_types))
5622 : {
5623 : found_mode = true;
5624 : break;
5625 : }
5626 : }
5627 :
5628 161 : if (!found_mode)
5629 0 : goto unsupported;
5630 :
5631 322 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5632 56 : codecvt2 = ERROR_MARK;
5633 : else
5634 : {
5635 105 : multi_step_cvt++;
5636 105 : interm_types.safe_push (cvt_type);
5637 105 : cvt_type = NULL_TREE;
5638 : }
5639 : break;
5640 :
5641 44825 : case NARROW_DST:
5642 44825 : gcc_assert (op_type == unary_op);
5643 44825 : if (supportable_narrowing_operation (code, vectype_out, vectype_in,
5644 : &code1, &multi_step_cvt,
5645 : &interm_types))
5646 : break;
5647 :
5648 19050 : if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
5649 986 : goto unsupported;
5650 :
5651 5364 : if (code == FIX_TRUNC_EXPR)
5652 : {
5653 107 : cvt_type
5654 107 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5655 107 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5656 107 : if (cvt_type == NULL_TREE)
5657 0 : goto unsupported;
5658 107 : if (supportable_convert_operation ((tree_code) code, cvt_type,
5659 : vectype_in))
5660 105 : codecvt1 = code;
5661 : else
5662 2 : goto unsupported;
5663 105 : if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
5664 : &code1, &multi_step_cvt,
5665 : &interm_types))
5666 : break;
5667 : }
5668 : /* If slp_op0 can be represented with low precision integer,
5669 : truncate it to cvt_type and the do FLOAT_EXPR. */
5670 5257 : else if (code == FLOAT_EXPR)
5671 : {
5672 792 : if (cost_vec)
5673 : {
5674 784 : wide_int op_min_value, op_max_value;
5675 784 : tree def;
5676 :
5677 : /* ??? Merge ranges in case of more than one lane. */
5678 784 : if (SLP_TREE_LANES (slp_op0) != 1
5679 133 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
5680 917 : || !vect_get_range_info (def, &op_min_value, &op_max_value))
5681 755 : goto unsupported;
5682 :
5683 29 : if ((wi::min_precision (op_max_value, SIGNED)
5684 29 : > GET_MODE_BITSIZE (lhs_mode))
5685 29 : || (wi::min_precision (op_min_value, SIGNED)
5686 27 : > GET_MODE_BITSIZE (lhs_mode)))
5687 2 : goto unsupported;
5688 784 : }
5689 :
5690 35 : cvt_type
5691 35 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
5692 35 : cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
5693 35 : if (cvt_type == NULL_TREE)
5694 0 : goto unsupported;
5695 35 : if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
5696 : &code1, &multi_step_cvt,
5697 : &interm_types))
5698 2 : goto unsupported;
5699 33 : if (supportable_convert_operation ((tree_code) code, vectype_out,
5700 : cvt_type))
5701 : {
5702 33 : codecvt1 = code;
5703 33 : modifier = NARROW_SRC;
5704 33 : break;
5705 : }
5706 : }
5707 :
5708 4469 : goto unsupported;
5709 :
5710 : default:
5711 : gcc_unreachable ();
5712 : }
5713 :
5714 110632 : if (modifier == WIDEN
5715 110632 : && loop_vinfo
5716 48923 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
5717 131642 : && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
5718 20988 : || widening_evenodd_fn_p (code1)))
5719 : {
5720 22 : if (dump_enabled_p ())
5721 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5722 : "can't use a fully-masked loop because"
5723 : " widening operation on even/odd elements"
5724 : " mixes up lanes.\n");
5725 22 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
5726 : }
5727 :
5728 110632 : if (cost_vec) /* transformation not required. */
5729 : {
5730 86607 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
5731 86607 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
5732 : {
5733 0 : if (dump_enabled_p ())
5734 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5735 : "incompatible vector types for invariants\n");
5736 : return false;
5737 : }
5738 86607 : DUMP_VECT_SCOPE ("vectorizable_conversion");
5739 86607 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
5740 86607 : if (modifier == NONE)
5741 : {
5742 17064 : SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
5743 17064 : vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
5744 : slp_node, cost_vec);
5745 : }
5746 69543 : else if (modifier == NARROW_SRC || modifier == NARROW_DST)
5747 : {
5748 28844 : SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
5749 : /* The final packing step produces one vector result per copy. */
5750 28844 : vect_model_promotion_demotion_cost (slp_node, nvectors,
5751 : multi_step_cvt, cost_vec,
5752 : widen_arith);
5753 : }
5754 : else
5755 : {
5756 40699 : SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
5757 : /* The initial unpacking step produces two vector results
5758 : per copy. MULTI_STEP_CVT is 0 for a single conversion,
5759 : so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5760 40699 : vect_model_promotion_demotion_cost (slp_node,
5761 : nvectors >> multi_step_cvt,
5762 : multi_step_cvt, cost_vec,
5763 : widen_arith);
5764 : }
5765 86607 : interm_types.release ();
5766 86607 : return true;
5767 : }
5768 :
5769 : /* Transform. */
5770 24025 : if (dump_enabled_p ())
5771 4441 : dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
5772 :
5773 : /* In case of multi-step conversion, we first generate conversion operations
5774 : to the intermediate types, and then from that types to the final one.
5775 : We create vector destinations for the intermediate type (TYPES) received
5776 : from supportable_*_operation, and store them in the correct order
5777 : for future use in vect_create_vectorized_*_stmts (). */
5778 48050 : auto_vec<tree> vec_dsts (multi_step_cvt + 1);
5779 24025 : bool widen_or_narrow_float_p
5780 24025 : = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
5781 24025 : vec_dest = vect_create_destination_var (scalar_dest,
5782 : widen_or_narrow_float_p
5783 : ? cvt_type : vectype_out);
5784 24025 : vec_dsts.quick_push (vec_dest);
5785 :
5786 24025 : if (multi_step_cvt)
5787 : {
5788 9383 : for (i = interm_types.length () - 1;
5789 9383 : interm_types.iterate (i, &intermediate_type); i--)
5790 : {
5791 4934 : vec_dest = vect_create_destination_var (scalar_dest,
5792 : intermediate_type);
5793 4934 : vec_dsts.quick_push (vec_dest);
5794 : }
5795 : }
5796 :
5797 24025 : if (cvt_type)
5798 76 : vec_dest = vect_create_destination_var (scalar_dest,
5799 : widen_or_narrow_float_p
5800 : ? vectype_out : cvt_type);
5801 :
5802 24025 : unsigned num_vectors = vect_get_num_copies (vinfo, slp_node);
5803 24025 : switch (modifier)
5804 : {
5805 4469 : case NONE:
5806 4469 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5807 : /* vec_dest is intermediate type operand when multi_step_cvt. */
5808 4469 : if (multi_step_cvt)
5809 : {
5810 21 : cvt_op = vec_dest;
5811 21 : vec_dest = vec_dsts[0];
5812 : }
5813 :
5814 9329 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5815 : {
5816 : /* Arguments are ready, create the new vector stmt. */
5817 4860 : gimple* new_stmt;
5818 4860 : if (multi_step_cvt)
5819 : {
5820 21 : gcc_assert (multi_step_cvt == 1);
5821 21 : new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
5822 21 : new_temp = make_ssa_name (cvt_op, new_stmt);
5823 21 : gimple_assign_set_lhs (new_stmt, new_temp);
5824 21 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5825 21 : vop0 = new_temp;
5826 : }
5827 4860 : if (code1 == COND_EXPR)
5828 : {
5829 1 : gcc_assert (!multi_step_cvt);
5830 1 : new_stmt
5831 2 : = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
5832 : build_minus_one_cst
5833 1 : (TREE_TYPE (vec_dest)),
5834 1 : build_zero_cst (TREE_TYPE (vec_dest)));
5835 1 : new_temp = make_ssa_name (vec_dest, new_stmt);
5836 1 : gimple_set_lhs (new_stmt, new_temp);
5837 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5838 1 : tree new_temp2 = make_ssa_name (vectype_out);
5839 1 : new_stmt = gimple_build_assign (new_temp2,
5840 : build1 (VIEW_CONVERT_EXPR,
5841 : vectype_out, new_temp));
5842 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5843 1 : slp_node->push_vec_def (new_temp2);
5844 : }
5845 : else
5846 : {
5847 4859 : new_stmt = vect_gimple_build (vec_dest, code1, vop0);
5848 4859 : new_temp = make_ssa_name (vec_dest, new_stmt);
5849 4859 : gimple_set_lhs (new_stmt, new_temp);
5850 4859 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5851 :
5852 4859 : slp_node->push_vec_def (new_stmt);
5853 : }
5854 : }
5855 : break;
5856 :
5857 9791 : case WIDEN:
5858 : /* In case the vectorization factor (VF) is bigger than the number
5859 : of elements that we can fit in a vectype (nunits), we have to
5860 : generate more than one vector stmt - i.e - we need to "unroll"
5861 : the vector stmt by a factor VF/nunits. */
5862 19582 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
5863 9791 : code != WIDEN_LSHIFT_EXPR && slp_op1, &vec_oprnds1);
5864 9791 : if (code == WIDEN_LSHIFT_EXPR)
5865 : {
5866 0 : int oprnds_size = vec_oprnds0.length ();
5867 0 : vec_oprnds1.create (oprnds_size);
5868 0 : for (i = 0; i < oprnds_size; ++i)
5869 0 : vec_oprnds1.quick_push (vect_get_slp_scalar_def (slp_op1, 0));
5870 : }
5871 : /* Arguments are ready. Create the new vector stmts. */
5872 21407 : for (i = multi_step_cvt; i >= 0; i--)
5873 : {
5874 11616 : tree this_dest = vec_dsts[i];
5875 11616 : code_helper c1 = code1, c2 = code2;
5876 11616 : if (i == 0 && codecvt2 != ERROR_MARK)
5877 : {
5878 48 : c1 = codecvt1;
5879 48 : c2 = codecvt2;
5880 : }
5881 11616 : if (known_eq (nunits_out, nunits_in))
5882 44 : vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
5883 : stmt_info, this_dest, gsi, c1,
5884 : op_type);
5885 : else
5886 : /* ??? For constant/external inputs we can end up with
5887 : excess lanes. When the number of inputs already match
5888 : the number of required outputs request half of the
5889 : lanes (gcc.dg/vect/O3-vect-pr32243.c). Low coverage
5890 : makes this likely incomplete. */
5891 13397 : vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
5892 : &vec_oprnds1, stmt_info,
5893 : this_dest, gsi,
5894 : c1, c2, op_type,
5895 : i == 0 ? num_vectors : -1u);
5896 : }
5897 :
5898 61393 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5899 : {
5900 27577 : gimple *new_stmt;
5901 27577 : if (cvt_type)
5902 : {
5903 120 : new_temp = make_ssa_name (vec_dest);
5904 120 : new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5905 120 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5906 : }
5907 : else
5908 27457 : new_stmt = SSA_NAME_DEF_STMT (vop0);
5909 :
5910 27577 : slp_node->push_vec_def (new_stmt);
5911 : }
5912 : break;
5913 :
5914 9765 : case NARROW_SRC:
5915 9765 : case NARROW_DST:
5916 : /* In case the vectorization factor (VF) is bigger than the number
5917 : of elements that we can fit in a vectype (nunits), we have to
5918 : generate more than one vector stmt - i.e - we need to "unroll"
5919 : the vector stmt by a factor VF/nunits. */
5920 9765 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5921 : /* Arguments are ready. Create the new vector stmts. */
5922 9765 : if (cvt_type && modifier == NARROW_DST)
5923 9918 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5924 : {
5925 124 : new_temp = make_ssa_name (vec_dest);
5926 124 : gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5927 124 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5928 124 : vec_oprnds0[i] = new_temp;
5929 : }
5930 :
5931 9765 : vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
5932 : multi_step_cvt,
5933 : stmt_info, vec_dsts, gsi,
5934 : slp_node, code1,
5935 : modifier == NARROW_SRC);
5936 : /* After demoting slp_op0 to cvt_type, convert it to dest. */
5937 9765 : if (cvt_type && code == FLOAT_EXPR)
5938 : {
5939 32 : for (unsigned int i = 0; i != vec_oprnds0.length() / 2; i++)
5940 : {
5941 : /* Arguments are ready, create the new vector stmt. */
5942 8 : gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
5943 8 : gimple *new_stmt
5944 8 : = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
5945 8 : new_temp = make_ssa_name (vec_dest, new_stmt);
5946 8 : gimple_set_lhs (new_stmt, new_temp);
5947 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5948 :
5949 : /* This is the last step of the conversion sequence. Store the
5950 : vectors in SLP_NODE or in vector info of the scalar statement
5951 : (or in STMT_VINFO_RELATED_STMT chain). */
5952 8 : slp_node->push_vec_def (new_stmt);
5953 : }
5954 : }
5955 : break;
5956 : }
5957 :
5958 24025 : vec_oprnds0.release ();
5959 24025 : vec_oprnds1.release ();
5960 24025 : interm_types.release ();
5961 :
5962 24025 : return true;
5963 194678 : }
5964 :
5965 : /* Return true if we can assume from the scalar form of STMT_INFO that
5966 : neither the scalar nor the vector forms will generate code. STMT_INFO
5967 : is known not to involve a data reference. */
5968 :
5969 : bool
5970 3315326 : vect_nop_conversion_p (stmt_vec_info stmt_info)
5971 : {
5972 3315326 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5973 3013261 : if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
5974 : return false;
5975 :
5976 991790 : tree lhs = gimple_assign_lhs (stmt);
5977 991790 : tree_code code = gimple_assign_rhs_code (stmt);
5978 991790 : tree rhs = gimple_assign_rhs1 (stmt);
5979 :
5980 991790 : if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
5981 : return true;
5982 :
5983 988452 : if (CONVERT_EXPR_CODE_P (code))
5984 247769 : return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
5985 :
5986 : return false;
5987 : }
5988 :
5989 : /* Function vectorizable_assignment.
5990 :
5991 : Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5992 : If COST_VEC is passed, calculate costs but don't change anything,
5993 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5994 : it, and insert it at GSI.
5995 : Return true if STMT_INFO is vectorizable in this way. */
5996 :
5997 : static bool
5998 2153639 : vectorizable_assignment (vec_info *vinfo,
5999 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6000 : slp_tree slp_node,
6001 : stmt_vector_for_cost *cost_vec)
6002 : {
6003 2153639 : tree vec_dest;
6004 2153639 : tree scalar_dest;
6005 2153639 : tree op;
6006 2153639 : tree new_temp;
6007 2153639 : enum vect_def_type dt[1] = {vect_unknown_def_type};
6008 2153639 : int i;
6009 2153639 : vec<tree> vec_oprnds = vNULL;
6010 2153639 : tree vop;
6011 2153639 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6012 2153639 : enum tree_code code;
6013 2153639 : tree vectype_in;
6014 :
6015 2153639 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6016 : return false;
6017 :
6018 2153639 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6019 247311 : && cost_vec)
6020 : return false;
6021 :
6022 : /* Is vectorizable assignment? */
6023 1906328 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6024 1830432 : if (!stmt)
6025 : return false;
6026 :
6027 1830432 : scalar_dest = gimple_assign_lhs (stmt);
6028 1830432 : if (TREE_CODE (scalar_dest) != SSA_NAME)
6029 : return false;
6030 :
6031 996616 : if (STMT_VINFO_DATA_REF (stmt_info))
6032 : return false;
6033 :
6034 420317 : code = gimple_assign_rhs_code (stmt);
6035 420317 : if (!(gimple_assign_single_p (stmt)
6036 418667 : || code == PAREN_EXPR
6037 417395 : || CONVERT_EXPR_CODE_P (code)))
6038 : return false;
6039 :
6040 105218 : tree vectype = SLP_TREE_VECTYPE (slp_node);
6041 105218 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6042 :
6043 105218 : slp_tree slp_op;
6044 105218 : if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
6045 : &dt[0], &vectype_in))
6046 : {
6047 0 : if (dump_enabled_p ())
6048 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6049 : "use not simple.\n");
6050 : return false;
6051 : }
6052 105218 : if (!vectype_in)
6053 20037 : vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
6054 :
6055 : /* We can handle VIEW_CONVERT conversions that do not change the number
6056 : of elements or the vector size or other conversions when the component
6057 : types are nop-convertible. */
6058 105218 : if (!vectype_in
6059 109869 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
6060 92746 : || (code == VIEW_CONVERT_EXPR
6061 3020 : && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
6062 3020 : GET_MODE_SIZE (TYPE_MODE (vectype_in))))
6063 197964 : || (CONVERT_EXPR_CODE_P (code)
6064 89856 : && !tree_nop_conversion_p (TREE_TYPE (vectype),
6065 89856 : TREE_TYPE (vectype_in))))
6066 : return false;
6067 :
6068 263832 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
6069 : {
6070 27 : if (dump_enabled_p ())
6071 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6072 : "can't convert between boolean and non "
6073 0 : "boolean vectors %T\n", TREE_TYPE (op));
6074 :
6075 : return false;
6076 : }
6077 :
6078 : /* We do not handle bit-precision changes. */
6079 88068 : if ((CONVERT_EXPR_CODE_P (code)
6080 2890 : || code == VIEW_CONVERT_EXPR)
6081 86688 : && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6082 85430 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6083 86142 : || (INTEGRAL_TYPE_P (TREE_TYPE (op))
6084 78140 : && !type_has_mode_precision_p (TREE_TYPE (op))))
6085 : /* But a conversion that does not change the bit-pattern is ok. */
6086 89180 : && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6087 1112 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
6088 1112 : && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
6089 1112 : > TYPE_PRECISION (TREE_TYPE (op)))
6090 566 : && TYPE_UNSIGNED (TREE_TYPE (op)))
6091 608 : || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
6092 608 : == TYPE_PRECISION (TREE_TYPE (op))))))
6093 : {
6094 377 : if (dump_enabled_p ())
6095 42 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6096 : "type conversion to/from bit-precision "
6097 : "unsupported.\n");
6098 : return false;
6099 : }
6100 :
6101 87691 : if (cost_vec) /* transformation not required. */
6102 : {
6103 71362 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
6104 : {
6105 0 : if (dump_enabled_p ())
6106 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6107 : "incompatible vector types for invariants\n");
6108 : return false;
6109 : }
6110 71362 : SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
6111 71362 : DUMP_VECT_SCOPE ("vectorizable_assignment");
6112 71362 : if (!vect_nop_conversion_p (stmt_info))
6113 1081 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6114 71362 : return true;
6115 : }
6116 :
6117 : /* Transform. */
6118 16329 : if (dump_enabled_p ())
6119 3760 : dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
6120 :
6121 : /* Handle def. */
6122 16329 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6123 :
6124 : /* Handle use. */
6125 16329 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
6126 :
6127 : /* Arguments are ready. create the new vector stmt. */
6128 53009 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
6129 : {
6130 20351 : gassign *new_stmt;
6131 20351 : if (code == PAREN_EXPR)
6132 510 : new_stmt = gimple_build_assign (vec_dest, PAREN_EXPR, vop);
6133 : else
6134 : {
6135 19841 : if (CONVERT_EXPR_CODE_P (code)
6136 204 : || code == VIEW_CONVERT_EXPR)
6137 19785 : vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
6138 19841 : new_stmt = gimple_build_assign (vec_dest, vop);
6139 : }
6140 20351 : new_temp = make_ssa_name (vec_dest, new_stmt);
6141 20351 : gimple_assign_set_lhs (new_stmt, new_temp);
6142 20351 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6143 20351 : slp_node->push_vec_def (new_stmt);
6144 : }
6145 :
6146 16329 : vec_oprnds.release ();
6147 16329 : return true;
6148 : }
6149 :
6150 :
6151 : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6152 : either as shift by a scalar or by a vector. */
6153 :
6154 : bool
6155 311901 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
6156 : {
6157 311901 : optab optab;
6158 311901 : tree vectype;
6159 :
6160 311901 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6161 311901 : if (!vectype)
6162 : return false;
6163 :
6164 311901 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6165 311901 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6166 : return true;
6167 :
6168 274182 : optab = optab_for_tree_code (code, vectype, optab_vector);
6169 274182 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6170 : return true;
6171 :
6172 : return false;
6173 : }
6174 :
6175 :
6176 : /* Function vectorizable_shift.
6177 :
6178 : Check if STMT_INFO performs a shift operation that can be vectorized.
6179 : If COST_VEC is passed, calculate costs but don't change anything,
6180 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6181 : it, and insert it at GSI.
6182 : Return true if STMT_INFO is vectorizable in this way. */
6183 :
6184 : static bool
6185 779534 : vectorizable_shift (vec_info *vinfo,
6186 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6187 : slp_tree slp_node,
6188 : stmt_vector_for_cost *cost_vec)
6189 : {
6190 779534 : tree vec_dest;
6191 779534 : tree scalar_dest;
6192 779534 : tree op0, op1 = NULL;
6193 779534 : tree vec_oprnd1 = NULL_TREE;
6194 779534 : tree vectype;
6195 779534 : enum tree_code code;
6196 779534 : machine_mode vec_mode;
6197 779534 : tree new_temp;
6198 779534 : optab optab;
6199 779534 : int icode;
6200 779534 : machine_mode optab_op2_mode;
6201 779534 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
6202 779534 : poly_uint64 nunits_in;
6203 779534 : poly_uint64 nunits_out;
6204 779534 : tree vectype_out;
6205 779534 : tree op1_vectype;
6206 779534 : int i;
6207 779534 : vec<tree> vec_oprnds0 = vNULL;
6208 779534 : vec<tree> vec_oprnds1 = vNULL;
6209 779534 : tree vop0, vop1;
6210 779534 : unsigned int k;
6211 779534 : bool scalar_shift_arg = true;
6212 779534 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6213 779534 : bool incompatible_op1_vectype_p = false;
6214 :
6215 779534 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6216 : return false;
6217 :
6218 779534 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6219 247311 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
6220 245932 : && cost_vec)
6221 : return false;
6222 :
6223 : /* Is STMT a vectorizable binary/unary operation? */
6224 533602 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6225 458796 : if (!stmt)
6226 : return false;
6227 :
6228 458796 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
6229 : return false;
6230 :
6231 458230 : code = gimple_assign_rhs_code (stmt);
6232 :
6233 458230 : if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
6234 : || code == RROTATE_EXPR))
6235 : return false;
6236 :
6237 61959 : scalar_dest = gimple_assign_lhs (stmt);
6238 61959 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6239 61959 : if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6240 : {
6241 0 : if (dump_enabled_p ())
6242 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6243 : "bit-precision shifts not supported.\n");
6244 : return false;
6245 : }
6246 :
6247 61959 : slp_tree slp_op0;
6248 61959 : if (!vect_is_simple_use (vinfo, slp_node,
6249 : 0, &op0, &slp_op0, &dt[0], &vectype))
6250 : {
6251 0 : if (dump_enabled_p ())
6252 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6253 : "use not simple.\n");
6254 : return false;
6255 : }
6256 : /* If op0 is an external or constant def, infer the vector type
6257 : from the scalar type. */
6258 61959 : if (!vectype)
6259 14538 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
6260 61959 : if (!cost_vec)
6261 9659 : gcc_assert (vectype);
6262 61959 : if (!vectype)
6263 : {
6264 0 : if (dump_enabled_p ())
6265 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6266 : "no vectype for scalar type\n");
6267 : return false;
6268 : }
6269 :
6270 61959 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6271 61959 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6272 61959 : if (maybe_ne (nunits_out, nunits_in))
6273 : return false;
6274 :
6275 61959 : stmt_vec_info op1_def_stmt_info;
6276 61959 : slp_tree slp_op1;
6277 61959 : if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
6278 : &dt[1], &op1_vectype, &op1_def_stmt_info))
6279 : {
6280 0 : if (dump_enabled_p ())
6281 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6282 : "use not simple.\n");
6283 : return false;
6284 : }
6285 :
6286 : /* Determine whether the shift amount is a vector, or scalar. If the
6287 : shift/rotate amount is a vector, use the vector/vector shift optabs. */
6288 :
6289 61959 : if ((dt[1] == vect_internal_def
6290 61959 : || dt[1] == vect_induction_def
6291 45334 : || dt[1] == vect_nested_cycle)
6292 16643 : && SLP_TREE_LANES (slp_node) == 1)
6293 : scalar_shift_arg = false;
6294 45375 : else if (dt[1] == vect_constant_def
6295 : || dt[1] == vect_external_def
6296 45375 : || dt[1] == vect_internal_def)
6297 : {
6298 : /* For internal SLP defs we have to make sure we see scalar stmts
6299 : for all vector elements.
6300 : ??? For different vectors we could resort to a different
6301 : scalar shift operand but code-generation below simply always
6302 : takes the first. */
6303 45369 : if (dt[1] == vect_internal_def
6304 45369 : && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
6305 : SLP_TREE_LANES (slp_node)))
6306 : scalar_shift_arg = false;
6307 : /* If the shift amount is computed by a pattern stmt we cannot
6308 : use the scalar amount directly thus give up and use a vector
6309 : shift. */
6310 45359 : else if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
6311 : scalar_shift_arg = false;
6312 : else
6313 : {
6314 45352 : tree l0 = vect_get_slp_scalar_def (slp_op1, 0);
6315 45352 : if (!l0)
6316 : scalar_shift_arg = false;
6317 : else
6318 70965 : for (k = 1; k < SLP_TREE_LANES (slp_node); ++k)
6319 25643 : if (!operand_equal_p (vect_get_slp_scalar_def (slp_op1, k), l0))
6320 : {
6321 : scalar_shift_arg = false;
6322 : break;
6323 : }
6324 : }
6325 : }
6326 : else
6327 : {
6328 6 : if (dump_enabled_p ())
6329 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6330 : "operand mode requires invariant argument.\n");
6331 : return false;
6332 : }
6333 :
6334 : /* Vector shifted by vector. */
6335 61983 : bool was_scalar_shift_arg = scalar_shift_arg;
6336 45352 : if (!scalar_shift_arg)
6337 : {
6338 16631 : optab = optab_for_tree_code (code, vectype, optab_vector);
6339 16631 : if (dump_enabled_p ())
6340 1235 : dump_printf_loc (MSG_NOTE, vect_location,
6341 : "vector/vector shift/rotate found.\n");
6342 :
6343 16631 : if (!op1_vectype)
6344 15 : op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
6345 : slp_op1);
6346 16631 : incompatible_op1_vectype_p
6347 33262 : = (op1_vectype == NULL_TREE
6348 16631 : || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
6349 16631 : TYPE_VECTOR_SUBPARTS (vectype))
6350 33260 : || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
6351 16631 : if (incompatible_op1_vectype_p
6352 7 : && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
6353 1 : || slp_op1->refcnt != 1))
6354 : {
6355 6 : if (dump_enabled_p ())
6356 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6357 : "unusable type for last operand in"
6358 : " vector/vector shift/rotate.\n");
6359 : return false;
6360 : }
6361 : }
6362 : /* See if the machine has a vector shifted by scalar insn and if not
6363 : then see if it has a vector shifted by vector insn. */
6364 : else
6365 : {
6366 45322 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6367 45322 : if (optab
6368 45322 : && can_implement_p (optab, TYPE_MODE (vectype)))
6369 : {
6370 45067 : if (dump_enabled_p ())
6371 5604 : dump_printf_loc (MSG_NOTE, vect_location,
6372 : "vector/scalar shift/rotate found.\n");
6373 : }
6374 : else
6375 : {
6376 255 : optab = optab_for_tree_code (code, vectype, optab_vector);
6377 255 : if (optab
6378 255 : && can_implement_p (optab, TYPE_MODE (vectype)))
6379 : {
6380 0 : scalar_shift_arg = false;
6381 :
6382 0 : if (dump_enabled_p ())
6383 0 : dump_printf_loc (MSG_NOTE, vect_location,
6384 : "vector/vector shift/rotate found.\n");
6385 :
6386 0 : if (!op1_vectype)
6387 0 : op1_vectype = get_vectype_for_scalar_type (vinfo,
6388 0 : TREE_TYPE (op1),
6389 : slp_op1);
6390 :
6391 : /* Unlike the other binary operators, shifts/rotates have
6392 : the rhs being int, instead of the same type as the lhs,
6393 : so make sure the scalar is the right type if we are
6394 : dealing with vectors of long long/long/short/char. */
6395 0 : incompatible_op1_vectype_p
6396 0 : = (!op1_vectype
6397 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype),
6398 0 : TREE_TYPE (op1)));
6399 : if (incompatible_op1_vectype_p
6400 0 : && dt[1] == vect_internal_def)
6401 : {
6402 0 : if (dump_enabled_p ())
6403 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6404 : "unusable type for last operand in"
6405 : " vector/vector shift/rotate.\n");
6406 : return false;
6407 : }
6408 : }
6409 : }
6410 : }
6411 :
6412 : /* Supportable by target? */
6413 61947 : if (!optab)
6414 : {
6415 0 : if (dump_enabled_p ())
6416 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6417 : "no shift optab for %s and %T.\n",
6418 : get_tree_code_name (code), vectype);
6419 : return false;
6420 : }
6421 61947 : vec_mode = TYPE_MODE (vectype);
6422 61947 : icode = (int) optab_handler (optab, vec_mode);
6423 61947 : if (icode == CODE_FOR_nothing)
6424 : {
6425 6545 : if (dump_enabled_p ())
6426 972 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6427 : "shift op not supported by target.\n");
6428 : return false;
6429 : }
6430 : /* vector lowering cannot optimize vector shifts using word arithmetic. */
6431 55402 : if (vect_emulated_vector_p (vectype))
6432 : return false;
6433 :
6434 55402 : if (cost_vec) /* transformation not required. */
6435 : {
6436 45743 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6437 45743 : || ((!scalar_shift_arg || dt[1] == vect_internal_def)
6438 8174 : && (!incompatible_op1_vectype_p
6439 1 : || dt[1] == vect_constant_def)
6440 8174 : && !vect_maybe_update_slp_op_vectype
6441 8174 : (slp_op1,
6442 : incompatible_op1_vectype_p ? vectype : op1_vectype)))
6443 : {
6444 0 : if (dump_enabled_p ())
6445 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6446 : "incompatible vector types for invariants\n");
6447 : return false;
6448 : }
6449 : /* Now adjust the constant shift amount in place. */
6450 45743 : if (incompatible_op1_vectype_p
6451 1 : && dt[1] == vect_constant_def)
6452 : {
6453 1 : unsigned group_size = SLP_TREE_LANES (slp_op1);
6454 2 : gcc_assert (SLP_TREE_SCALAR_OPS (slp_op1).length () == group_size);
6455 5 : for (unsigned i = 0; i < group_size; ++i)
6456 : {
6457 4 : SLP_TREE_SCALAR_OPS (slp_op1)[i]
6458 4 : = fold_convert (TREE_TYPE (vectype),
6459 : SLP_TREE_SCALAR_OPS (slp_op1)[i]);
6460 4 : gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
6461 : == INTEGER_CST));
6462 : }
6463 : }
6464 45743 : SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
6465 45743 : DUMP_VECT_SCOPE ("vectorizable_shift");
6466 45743 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6467 45743 : return true;
6468 : }
6469 :
6470 : /* Transform. */
6471 :
6472 9659 : if (dump_enabled_p ())
6473 2268 : dump_printf_loc (MSG_NOTE, vect_location,
6474 : "transform binary/unary operation.\n");
6475 :
6476 : /* Handle def. */
6477 9659 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6478 :
6479 9659 : unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
6480 9659 : if (scalar_shift_arg && dt[1] != vect_internal_def)
6481 : {
6482 : /* Vector shl and shr insn patterns can be defined with scalar
6483 : operand 2 (shift operand). In this case, use constant or loop
6484 : invariant op1 directly, without extending it to vector mode
6485 : first. */
6486 7474 : optab_op2_mode = insn_data[icode].operand[2].mode;
6487 7474 : if (!VECTOR_MODE_P (optab_op2_mode))
6488 : {
6489 7474 : if (dump_enabled_p ())
6490 2151 : dump_printf_loc (MSG_NOTE, vect_location,
6491 : "operand 1 using scalar mode.\n");
6492 7474 : vec_oprnd1 = op1;
6493 7474 : vec_oprnds1.create (nvectors);
6494 7474 : vec_oprnds1.quick_push (vec_oprnd1);
6495 : /* Store vec_oprnd1 for every vector stmt to be created.
6496 : We check during the analysis that all the shift arguments
6497 : are the same.
6498 : TODO: Allow different constants for different vector
6499 : stmts generated for an SLP instance. */
6500 17542 : for (k = 0; k < nvectors - 1; k++)
6501 2594 : vec_oprnds1.quick_push (vec_oprnd1);
6502 : }
6503 : }
6504 2185 : else if (!scalar_shift_arg && incompatible_op1_vectype_p)
6505 : {
6506 0 : if (was_scalar_shift_arg)
6507 : {
6508 : /* If the argument was the same in all lanes create the
6509 : correctly typed vector shift amount directly. Note
6510 : we made SLP scheduling think we use the original scalars,
6511 : so place the compensation code next to the shift which
6512 : is conservative. See PR119640 where it otherwise breaks. */
6513 0 : op1 = fold_convert (TREE_TYPE (vectype), op1);
6514 0 : op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
6515 : gsi);
6516 0 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
6517 : gsi);
6518 0 : vec_oprnds1.create (nvectors);
6519 0 : for (k = 0; k < nvectors; k++)
6520 0 : vec_oprnds1.quick_push (vec_oprnd1);
6521 : }
6522 0 : else if (dt[1] == vect_constant_def)
6523 : /* The constant shift amount has been adjusted in place. */
6524 : ;
6525 : else
6526 0 : gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
6527 : }
6528 :
6529 : /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6530 : (a special case for certain kind of vector shifts); otherwise,
6531 : operand 1 should be of a vector type (the usual case). */
6532 9659 : vect_get_vec_defs (vinfo, slp_node,
6533 : true, &vec_oprnds0, !vec_oprnd1, &vec_oprnds1);
6534 :
6535 : /* Arguments are ready. Create the new vector stmt. */
6536 35105 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6537 : {
6538 : /* For internal defs where we need to use a scalar shift arg
6539 : extract the first lane. */
6540 15787 : if (scalar_shift_arg && dt[1] == vect_internal_def)
6541 : {
6542 12 : vop1 = vec_oprnds1[0];
6543 12 : new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
6544 12 : gassign *new_stmt
6545 12 : = gimple_build_assign (new_temp,
6546 12 : build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
6547 : vop1,
6548 12 : TYPE_SIZE (TREE_TYPE (new_temp)),
6549 : bitsize_zero_node));
6550 12 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6551 12 : vop1 = new_temp;
6552 12 : }
6553 : else
6554 15775 : vop1 = vec_oprnds1[i];
6555 15787 : gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
6556 15787 : new_temp = make_ssa_name (vec_dest, new_stmt);
6557 15787 : gimple_assign_set_lhs (new_stmt, new_temp);
6558 15787 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6559 15787 : slp_node->push_vec_def (new_stmt);
6560 : }
6561 :
6562 9659 : vec_oprnds0.release ();
6563 9659 : vec_oprnds1.release ();
6564 :
6565 9659 : return true;
6566 : }
6567 :
6568 : /* Function vectorizable_operation.
6569 :
6570 : Check if STMT_INFO performs a binary, unary or ternary operation that can
6571 : be vectorized.
6572 : If COST_VEC is passed, calculate costs but don't change anything,
6573 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6574 : it, and insert it at GSI.
6575 : Return true if STMT_INFO is vectorizable in this way. */
6576 :
6577 : static bool
6578 2687247 : vectorizable_operation (vec_info *vinfo,
6579 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6580 : slp_tree slp_node,
6581 : stmt_vector_for_cost *cost_vec)
6582 : {
6583 2687247 : tree vec_dest;
6584 2687247 : tree scalar_dest;
6585 2687247 : tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
6586 2687247 : tree vectype;
6587 2687247 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6588 2687247 : enum tree_code code, orig_code;
6589 2687247 : machine_mode vec_mode;
6590 2687247 : tree new_temp;
6591 2687247 : int op_type;
6592 2687247 : optab optab;
6593 2687247 : bool target_support_p;
6594 2687247 : enum vect_def_type dt[3]
6595 : = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
6596 2687247 : poly_uint64 nunits_in;
6597 2687247 : poly_uint64 nunits_out;
6598 2687247 : tree vectype_out;
6599 2687247 : int i;
6600 2687247 : vec<tree> vec_oprnds0 = vNULL;
6601 2687247 : vec<tree> vec_oprnds1 = vNULL;
6602 2687247 : vec<tree> vec_oprnds2 = vNULL;
6603 2687247 : tree vop0, vop1, vop2;
6604 2687247 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6605 :
6606 2687247 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6607 : return false;
6608 :
6609 2687247 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6610 247311 : && cost_vec)
6611 : return false;
6612 :
6613 : /* Is STMT a vectorizable binary/unary operation? */
6614 2439936 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6615 2364040 : if (!stmt)
6616 : return false;
6617 :
6618 : /* Loads and stores are handled in vectorizable_{load,store}. */
6619 2364040 : if (STMT_VINFO_DATA_REF (stmt_info))
6620 : return false;
6621 :
6622 953925 : orig_code = code = gimple_assign_rhs_code (stmt);
6623 :
6624 : /* Shifts are handled in vectorizable_shift. */
6625 953925 : if (code == LSHIFT_EXPR
6626 : || code == RSHIFT_EXPR
6627 : || code == LROTATE_EXPR
6628 953925 : || code == RROTATE_EXPR)
6629 : return false;
6630 :
6631 : /* Comparisons are handled in vectorizable_comparison. */
6632 901625 : if (TREE_CODE_CLASS (code) == tcc_comparison)
6633 : return false;
6634 :
6635 : /* Conditions are handled in vectorizable_condition. */
6636 700270 : if (code == COND_EXPR)
6637 : return false;
6638 :
6639 : /* For pointer addition and subtraction, we should use the normal
6640 : plus and minus for the vector operation. */
6641 673090 : if (code == POINTER_PLUS_EXPR)
6642 : code = PLUS_EXPR;
6643 652212 : if (code == POINTER_DIFF_EXPR)
6644 3017 : code = MINUS_EXPR;
6645 :
6646 : /* Support only unary or binary operations. */
6647 673090 : op_type = TREE_CODE_LENGTH (code);
6648 673090 : if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
6649 : {
6650 12 : if (dump_enabled_p ())
6651 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6652 : "num. args = %d (not unary/binary/ternary op).\n",
6653 : op_type);
6654 : return false;
6655 : }
6656 :
6657 673078 : scalar_dest = gimple_assign_lhs (stmt);
6658 673078 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6659 :
6660 : /* Most operations cannot handle bit-precision types without extra
6661 : truncations. */
6662 673078 : bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
6663 661523 : if (!mask_op_p
6664 661523 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
6665 : /* Exception are bitwise binary operations. */
6666 : && code != BIT_IOR_EXPR
6667 2038 : && code != BIT_XOR_EXPR
6668 1169 : && code != BIT_AND_EXPR)
6669 : {
6670 837 : if (dump_enabled_p ())
6671 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6672 : "bit-precision arithmetic not supported.\n");
6673 : return false;
6674 : }
6675 :
6676 672241 : slp_tree slp_op0;
6677 672241 : if (!vect_is_simple_use (vinfo, slp_node,
6678 : 0, &op0, &slp_op0, &dt[0], &vectype))
6679 : {
6680 0 : if (dump_enabled_p ())
6681 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6682 : "use not simple.\n");
6683 : return false;
6684 : }
6685 672241 : bool is_invariant = (dt[0] == vect_external_def
6686 672241 : || dt[0] == vect_constant_def);
6687 : /* If op0 is an external or constant def, infer the vector type
6688 : from the scalar type. */
6689 672241 : if (!vectype)
6690 : {
6691 : /* For boolean type we cannot determine vectype by
6692 : invariant value (don't know whether it is a vector
6693 : of booleans or vector of integers). We use output
6694 : vectype because operations on boolean don't change
6695 : type. */
6696 78428 : if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
6697 : {
6698 2175 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
6699 : {
6700 334 : if (dump_enabled_p ())
6701 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6702 : "not supported operation on bool value.\n");
6703 : return false;
6704 : }
6705 1841 : vectype = vectype_out;
6706 : }
6707 : else
6708 76253 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
6709 : slp_node);
6710 : }
6711 671907 : if (!cost_vec)
6712 121018 : gcc_assert (vectype);
6713 671907 : if (!vectype)
6714 : {
6715 308 : if (dump_enabled_p ())
6716 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6717 : "no vectype for scalar type %T\n",
6718 2 : TREE_TYPE (op0));
6719 :
6720 : return false;
6721 : }
6722 :
6723 671599 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6724 671599 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6725 671599 : if (maybe_ne (nunits_out, nunits_in)
6726 671599 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
6727 : return false;
6728 :
6729 650556 : tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
6730 650556 : slp_tree slp_op1 = NULL, slp_op2 = NULL;
6731 650556 : if (op_type == binary_op || op_type == ternary_op)
6732 : {
6733 568133 : if (!vect_is_simple_use (vinfo, slp_node,
6734 : 1, &op1, &slp_op1, &dt[1], &vectype2))
6735 : {
6736 0 : if (dump_enabled_p ())
6737 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6738 : "use not simple.\n");
6739 : return false;
6740 : }
6741 568133 : is_invariant &= (dt[1] == vect_external_def
6742 568133 : || dt[1] == vect_constant_def);
6743 568133 : if (vectype2
6744 920642 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
6745 352509 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6746 352509 : TREE_TYPE (vectype2))))
6747 : return false;
6748 : }
6749 650556 : if (op_type == ternary_op)
6750 : {
6751 0 : if (!vect_is_simple_use (vinfo, slp_node,
6752 : 2, &op2, &slp_op2, &dt[2], &vectype3))
6753 : {
6754 0 : if (dump_enabled_p ())
6755 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6756 : "use not simple.\n");
6757 : return false;
6758 : }
6759 0 : is_invariant &= (dt[2] == vect_external_def
6760 0 : || dt[2] == vect_constant_def);
6761 0 : if (vectype3
6762 0 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
6763 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6764 0 : TREE_TYPE (vectype3))))
6765 : return false;
6766 : }
6767 :
6768 : /* Multiple types in SLP are handled by creating the appropriate number of
6769 : vectorized stmts for each SLP node. */
6770 650556 : auto vec_num = vect_get_num_copies (vinfo, slp_node);
6771 :
6772 : /* Reject attempts to combine mask types with nonmask types, e.g. if
6773 : we have an AND between a (nonmask) boolean loaded from memory and
6774 : a (mask) boolean result of a comparison.
6775 :
6776 : TODO: We could easily fix these cases up using pattern statements. */
6777 650556 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
6778 994928 : || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
6779 1301112 : || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
6780 : {
6781 0 : if (dump_enabled_p ())
6782 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6783 : "mixed mask and nonmask vector types\n");
6784 : return false;
6785 : }
6786 :
6787 : /* Supportable by target? */
6788 :
6789 650556 : vec_mode = TYPE_MODE (vectype);
6790 650556 : optab = optab_for_tree_code (code, vectype, optab_default);
6791 650556 : if (!optab)
6792 : {
6793 70660 : if (dump_enabled_p ())
6794 6078 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6795 : "no optab for %s and %T.\n",
6796 : get_tree_code_name (code), vectype);
6797 : return false;
6798 : }
6799 579896 : target_support_p = can_implement_p (optab, vec_mode);
6800 :
6801 579896 : bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
6802 579896 : if (!target_support_p || using_emulated_vectors_p)
6803 : {
6804 30393 : if (dump_enabled_p ())
6805 1290 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6806 : "op not supported by target.\n");
6807 : /* When vec_mode is not a vector mode and we verified ops we
6808 : do not have to lower like AND are natively supported let
6809 : those through even when the mode isn't word_mode. For
6810 : ops we have to lower the lowering code assumes we are
6811 : dealing with word_mode. */
6812 60786 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
6813 30251 : || !GET_MODE_SIZE (vec_mode).is_constant ()
6814 30251 : || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6815 25053 : || !target_support_p)
6816 64996 : && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
6817 : /* Check only during analysis. */
6818 42606 : || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
6819 : {
6820 29789 : if (dump_enabled_p ())
6821 1288 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
6822 : return false;
6823 : }
6824 604 : if (dump_enabled_p ())
6825 2 : dump_printf_loc (MSG_NOTE, vect_location,
6826 : "proceeding using word mode.\n");
6827 : using_emulated_vectors_p = true;
6828 : }
6829 :
6830 550107 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
6831 550107 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
6832 446633 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
6833 550107 : internal_fn cond_fn = get_conditional_internal_fn (code);
6834 550107 : internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
6835 :
6836 : /* If operating on inactive elements could generate spurious traps,
6837 : we need to restrict the operation to active lanes. Note that this
6838 : specifically doesn't apply to unhoisted invariants, since they
6839 : operate on the same value for every lane.
6840 :
6841 : Similarly, if this operation is part of a reduction, a fully-masked
6842 : loop should only change the active lanes of the reduction chain,
6843 : keeping the inactive lanes as-is. */
6844 521500 : bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
6845 1002229 : || reduc_idx >= 0);
6846 :
6847 550107 : if (cost_vec) /* transformation not required. */
6848 : {
6849 429089 : if (loop_vinfo
6850 338343 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
6851 91945 : && mask_out_inactive)
6852 : {
6853 20401 : if (cond_len_fn != IFN_LAST
6854 20401 : && direct_internal_fn_supported_p (cond_len_fn, vectype,
6855 : OPTIMIZE_FOR_SPEED))
6856 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
6857 : 1);
6858 20401 : else if (cond_fn != IFN_LAST
6859 20401 : && direct_internal_fn_supported_p (cond_fn, vectype,
6860 : OPTIMIZE_FOR_SPEED))
6861 8505 : vect_record_loop_mask (loop_vinfo, masks, vec_num,
6862 : vectype, NULL);
6863 : else
6864 : {
6865 11896 : if (dump_enabled_p ())
6866 606 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6867 : "can't use a fully-masked loop because no"
6868 : " conditional operation is available.\n");
6869 11896 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6870 : }
6871 : }
6872 :
6873 : /* Put types on constant and invariant SLP children. */
6874 429089 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6875 429013 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
6876 858008 : || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
6877 : {
6878 170 : if (dump_enabled_p ())
6879 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6880 : "incompatible vector types for invariants\n");
6881 : return false;
6882 : }
6883 :
6884 428919 : SLP_TREE_TYPE (slp_node) = op_vec_info_type;
6885 428919 : DUMP_VECT_SCOPE ("vectorizable_operation");
6886 428919 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6887 428919 : if (using_emulated_vectors_p)
6888 : {
6889 : /* The above vect_model_simple_cost call handles constants
6890 : in the prologue and (mis-)costs one of the stmts as
6891 : vector stmt. See below for the actual lowering that will
6892 : be applied. */
6893 602 : unsigned n = vect_get_num_copies (vinfo, slp_node);
6894 602 : switch (code)
6895 : {
6896 213 : case PLUS_EXPR:
6897 213 : n *= 5;
6898 213 : break;
6899 352 : case MINUS_EXPR:
6900 352 : n *= 6;
6901 352 : break;
6902 0 : case NEGATE_EXPR:
6903 0 : n *= 4;
6904 0 : break;
6905 : default:
6906 : /* Bit operations do not have extra cost and are accounted
6907 : as vector stmt by vect_model_simple_cost. */
6908 : n = 0;
6909 : break;
6910 : }
6911 565 : if (n != 0)
6912 : {
6913 : /* We also need to materialize two large constants. */
6914 565 : record_stmt_cost (cost_vec, 2, scalar_stmt, slp_node,
6915 : 0, vect_prologue);
6916 565 : record_stmt_cost (cost_vec, n, scalar_stmt, slp_node,
6917 : 0, vect_body);
6918 : }
6919 : }
6920 428919 : return true;
6921 : }
6922 :
6923 : /* Transform. */
6924 :
6925 121018 : if (dump_enabled_p ())
6926 17312 : dump_printf_loc (MSG_NOTE, vect_location,
6927 : "transform binary/unary operation.\n");
6928 :
6929 121018 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
6930 108290 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
6931 :
6932 : /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6933 : vectors with unsigned elements, but the result is signed. So, we
6934 : need to compute the MINUS_EXPR into vectype temporary and
6935 : VIEW_CONVERT_EXPR it into the final vectype_out result. */
6936 121018 : tree vec_cvt_dest = NULL_TREE;
6937 121018 : if (orig_code == POINTER_DIFF_EXPR)
6938 : {
6939 111 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6940 111 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6941 : }
6942 : /* For reduction operations with undefined overflow behavior make sure to
6943 : pun them to unsigned since we change the order of evaluation.
6944 : ??? Avoid for in-order reductions? */
6945 120907 : else if (arith_code_with_undefined_signed_overflow (orig_code)
6946 101562 : && ANY_INTEGRAL_TYPE_P (vectype)
6947 50088 : && TYPE_OVERFLOW_UNDEFINED (vectype)
6948 146913 : && SLP_TREE_REDUC_IDX (slp_node) != -1)
6949 : {
6950 2463 : gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
6951 : || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
6952 2463 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6953 2463 : vectype = unsigned_type_for (vectype);
6954 2463 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6955 : }
6956 : /* Handle def. */
6957 : else
6958 118444 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
6959 :
6960 121018 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
6961 : slp_op1, &vec_oprnds1, slp_op2, &vec_oprnds2);
6962 : /* Arguments are ready. Create the new vector stmt. */
6963 389516 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6964 : {
6965 147480 : gimple *new_stmt = NULL;
6966 294960 : vop1 = ((op_type == binary_op || op_type == ternary_op)
6967 147480 : ? vec_oprnds1[i] : NULL_TREE);
6968 147480 : vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
6969 :
6970 147480 : if (vec_cvt_dest
6971 147480 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
6972 : {
6973 2922 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
6974 2922 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6975 : new_temp);
6976 2922 : new_temp = make_ssa_name (vec_dest, new_stmt);
6977 2922 : gimple_assign_set_lhs (new_stmt, new_temp);
6978 2922 : vect_finish_stmt_generation (vinfo, stmt_info,
6979 : new_stmt, gsi);
6980 2922 : vop0 = new_temp;
6981 : }
6982 147480 : if (vop1
6983 144740 : && vec_cvt_dest
6984 150528 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
6985 : {
6986 2922 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
6987 2922 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6988 : new_temp);
6989 2922 : new_temp = make_ssa_name (vec_dest, new_stmt);
6990 2922 : gimple_assign_set_lhs (new_stmt, new_temp);
6991 2922 : vect_finish_stmt_generation (vinfo, stmt_info,
6992 : new_stmt, gsi);
6993 2922 : vop1 = new_temp;
6994 : }
6995 147480 : if (vop2
6996 0 : && vec_cvt_dest
6997 147480 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
6998 : {
6999 0 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
7000 0 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
7001 : new_temp);
7002 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
7003 0 : gimple_assign_set_lhs (new_stmt, new_temp);
7004 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7005 : new_stmt, gsi);
7006 0 : vop2 = new_temp;
7007 : }
7008 :
7009 147480 : if (using_emulated_vectors_p)
7010 : {
7011 : /* Lower the operation. This follows vector lowering. */
7012 2 : tree word_type = build_nonstandard_integer_type
7013 2 : (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
7014 2 : tree wvop0 = make_ssa_name (word_type);
7015 2 : new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
7016 : build1 (VIEW_CONVERT_EXPR,
7017 : word_type, vop0));
7018 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7019 2 : tree wvop1 = NULL_TREE;
7020 2 : if (vop1)
7021 : {
7022 2 : wvop1 = make_ssa_name (word_type);
7023 2 : new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
7024 : build1 (VIEW_CONVERT_EXPR,
7025 : word_type, vop1));
7026 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7027 : }
7028 :
7029 2 : tree result_low;
7030 2 : if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
7031 : {
7032 1 : unsigned int width = vector_element_bits (vectype);
7033 1 : tree inner_type = TREE_TYPE (vectype);
7034 1 : HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
7035 1 : tree low_bits
7036 1 : = build_replicated_int_cst (word_type, width, max >> 1);
7037 1 : tree high_bits
7038 2 : = build_replicated_int_cst (word_type,
7039 1 : width, max & ~(max >> 1));
7040 1 : tree signs;
7041 1 : if (code == PLUS_EXPR || code == MINUS_EXPR)
7042 : {
7043 1 : signs = make_ssa_name (word_type);
7044 1 : new_stmt = gimple_build_assign (signs,
7045 : BIT_XOR_EXPR, wvop0, wvop1);
7046 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7047 1 : tree b_low = make_ssa_name (word_type);
7048 1 : new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
7049 : wvop1, low_bits);
7050 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7051 1 : tree a_low = make_ssa_name (word_type);
7052 1 : if (code == PLUS_EXPR)
7053 1 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7054 : wvop0, low_bits);
7055 : else
7056 0 : new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
7057 : wvop0, high_bits);
7058 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7059 1 : if (code == MINUS_EXPR)
7060 : {
7061 0 : new_stmt = gimple_build_assign (NULL_TREE,
7062 : BIT_NOT_EXPR, signs);
7063 0 : signs = make_ssa_name (word_type);
7064 0 : gimple_assign_set_lhs (new_stmt, signs);
7065 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7066 : new_stmt, gsi);
7067 : }
7068 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7069 : signs, high_bits);
7070 1 : signs = make_ssa_name (word_type);
7071 1 : gimple_assign_set_lhs (new_stmt, signs);
7072 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7073 1 : result_low = make_ssa_name (word_type);
7074 1 : new_stmt = gimple_build_assign (result_low, code,
7075 : a_low, b_low);
7076 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7077 : }
7078 : else /* if (code == NEGATE_EXPR) */
7079 : {
7080 0 : tree a_low = make_ssa_name (word_type);
7081 0 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7082 : wvop0, low_bits);
7083 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7084 0 : signs = make_ssa_name (word_type);
7085 0 : new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
7086 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7087 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7088 : signs, high_bits);
7089 0 : signs = make_ssa_name (word_type);
7090 0 : gimple_assign_set_lhs (new_stmt, signs);
7091 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7092 0 : result_low = make_ssa_name (word_type);
7093 0 : new_stmt = gimple_build_assign (result_low,
7094 : MINUS_EXPR, high_bits, a_low);
7095 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7096 : }
7097 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
7098 : result_low, signs);
7099 1 : result_low = make_ssa_name (word_type);
7100 1 : gimple_assign_set_lhs (new_stmt, result_low);
7101 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7102 : }
7103 : else
7104 : {
7105 1 : new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
7106 1 : result_low = make_ssa_name (word_type);
7107 1 : gimple_assign_set_lhs (new_stmt, result_low);
7108 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7109 :
7110 : }
7111 2 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
7112 : build1 (VIEW_CONVERT_EXPR,
7113 : vectype, result_low));
7114 2 : new_temp = make_ssa_name (vectype);
7115 2 : gimple_assign_set_lhs (new_stmt, new_temp);
7116 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7117 : }
7118 147478 : else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
7119 : {
7120 16 : tree mask;
7121 16 : if (masked_loop_p)
7122 16 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7123 : vec_num, vectype, i);
7124 : else
7125 : /* Dummy mask. */
7126 0 : mask = build_minus_one_cst (truth_type_for (vectype));
7127 16 : auto_vec<tree> vops (6);
7128 16 : vops.quick_push (mask);
7129 16 : vops.quick_push (vop0);
7130 16 : if (vop1)
7131 16 : vops.quick_push (vop1);
7132 16 : if (vop2)
7133 0 : vops.quick_push (vop2);
7134 16 : if (reduc_idx >= 0)
7135 : {
7136 : /* Perform the operation on active elements only and take
7137 : inactive elements from the reduction chain input. */
7138 8 : gcc_assert (!vop2);
7139 8 : vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
7140 : }
7141 : else
7142 : {
7143 8 : auto else_value = targetm.preferred_else_value
7144 8 : (cond_fn, vectype, vops.length () - 1, &vops[1]);
7145 8 : vops.quick_push (else_value);
7146 : }
7147 16 : if (len_loop_p)
7148 : {
7149 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
7150 0 : vec_num, vectype, i, 1, true);
7151 0 : signed char biasval
7152 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
7153 0 : tree bias = build_int_cst (intQI_type_node, biasval);
7154 0 : vops.quick_push (len);
7155 0 : vops.quick_push (bias);
7156 : }
7157 16 : gcall *call
7158 16 : = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
7159 : : cond_len_fn,
7160 : vops);
7161 16 : new_temp = make_ssa_name (vec_dest, call);
7162 16 : gimple_call_set_lhs (call, new_temp);
7163 16 : gimple_call_set_nothrow (call, true);
7164 16 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
7165 16 : new_stmt = call;
7166 16 : }
7167 : else
7168 : {
7169 147462 : tree mask = NULL_TREE;
7170 : /* When combining two masks check if either of them is elsewhere
7171 : combined with a loop mask, if that's the case we can mark that the
7172 : new combined mask doesn't need to be combined with a loop mask. */
7173 147462 : if (masked_loop_p
7174 147462 : && code == BIT_AND_EXPR
7175 147462 : && VECTOR_BOOLEAN_TYPE_P (vectype))
7176 : {
7177 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
7178 : {
7179 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7180 : vec_num, vectype, i);
7181 :
7182 0 : vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7183 : vop0, gsi);
7184 : }
7185 :
7186 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
7187 : {
7188 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7189 : vec_num, vectype, i);
7190 :
7191 0 : vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7192 : vop1, gsi);
7193 : }
7194 : }
7195 :
7196 147462 : new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
7197 147462 : new_temp = make_ssa_name (vec_dest, new_stmt);
7198 147462 : gimple_assign_set_lhs (new_stmt, new_temp);
7199 147462 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7200 147462 : if (using_emulated_vectors_p)
7201 : suppress_warning (new_stmt, OPT_Wvector_operation_performance);
7202 :
7203 : /* Enter the combined value into the vector cond hash so we don't
7204 : AND it with a loop mask again. */
7205 147462 : if (mask)
7206 0 : loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
7207 : }
7208 :
7209 147480 : if (vec_cvt_dest)
7210 : {
7211 3048 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
7212 3048 : new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
7213 : new_temp);
7214 3048 : new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
7215 3048 : gimple_assign_set_lhs (new_stmt, new_temp);
7216 3048 : vect_finish_stmt_generation (vinfo, stmt_info,
7217 : new_stmt, gsi);
7218 : }
7219 :
7220 147480 : slp_node->push_vec_def (new_stmt);
7221 : }
7222 :
7223 121018 : vec_oprnds0.release ();
7224 121018 : vec_oprnds1.release ();
7225 121018 : vec_oprnds2.release ();
7226 :
7227 121018 : return true;
7228 : }
7229 :
7230 : /* A helper function to ensure data reference DR_INFO's base alignment. */
7231 :
7232 : static void
7233 706766 : ensure_base_align (dr_vec_info *dr_info)
7234 : {
7235 : /* Alignment is only analyzed for the first element of a DR group,
7236 : use that to look at base alignment we need to enforce. */
7237 706766 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
7238 583378 : dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
7239 :
7240 706766 : gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
7241 :
7242 706766 : if (dr_info->base_misaligned)
7243 : {
7244 113516 : tree base_decl = dr_info->base_decl;
7245 :
7246 : // We should only be able to increase the alignment of a base object if
7247 : // we know what its new alignment should be at compile time.
7248 113516 : unsigned HOST_WIDE_INT align_base_to =
7249 113516 : DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
7250 :
7251 113516 : if (decl_in_symtab_p (base_decl))
7252 3242 : symtab_node::get (base_decl)->increase_alignment (align_base_to);
7253 110274 : else if (DECL_ALIGN (base_decl) < align_base_to)
7254 : {
7255 91954 : SET_DECL_ALIGN (base_decl, align_base_to);
7256 91954 : DECL_USER_ALIGN (base_decl) = 1;
7257 : }
7258 113516 : dr_info->base_misaligned = false;
7259 : }
7260 706766 : }
7261 :
7262 :
7263 : /* Function get_group_alias_ptr_type.
7264 :
7265 : Return the alias type for the group starting at FIRST_STMT_INFO. */
7266 :
7267 : static tree
7268 1658875 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
7269 : {
7270 1658875 : struct data_reference *first_dr, *next_dr;
7271 :
7272 1658875 : first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
7273 1658875 : stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
7274 3928870 : while (next_stmt_info)
7275 : {
7276 2459701 : next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
7277 4919402 : if (get_alias_set (DR_REF (first_dr))
7278 2459701 : != get_alias_set (DR_REF (next_dr)))
7279 : {
7280 189706 : if (dump_enabled_p ())
7281 30 : dump_printf_loc (MSG_NOTE, vect_location,
7282 : "conflicting alias set types.\n");
7283 189706 : return ptr_type_node;
7284 : }
7285 2269995 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7286 : }
7287 1469169 : return reference_alias_ptr_type (DR_REF (first_dr));
7288 : }
7289 :
7290 :
7291 : /* Function scan_operand_equal_p.
7292 :
7293 : Helper function for check_scan_store. Compare two references
7294 : with .GOMP_SIMD_LANE bases. */
7295 :
7296 : static bool
7297 1284 : scan_operand_equal_p (tree ref1, tree ref2)
7298 : {
7299 1284 : tree ref[2] = { ref1, ref2 };
7300 1284 : poly_int64 bitsize[2], bitpos[2];
7301 : tree offset[2], base[2];
7302 3852 : for (int i = 0; i < 2; ++i)
7303 : {
7304 2568 : machine_mode mode;
7305 2568 : int unsignedp, reversep, volatilep = 0;
7306 2568 : base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
7307 : &offset[i], &mode, &unsignedp,
7308 : &reversep, &volatilep);
7309 2568 : if (reversep || volatilep || maybe_ne (bitpos[i], 0))
7310 0 : return false;
7311 2568 : if (TREE_CODE (base[i]) == MEM_REF
7312 42 : && offset[i] == NULL_TREE
7313 2610 : && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
7314 : {
7315 42 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
7316 42 : if (is_gimple_assign (def_stmt)
7317 42 : && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
7318 42 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
7319 84 : && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
7320 : {
7321 42 : if (maybe_ne (mem_ref_offset (base[i]), 0))
7322 : return false;
7323 42 : base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
7324 42 : offset[i] = gimple_assign_rhs2 (def_stmt);
7325 : }
7326 : }
7327 : }
7328 :
7329 1284 : if (!operand_equal_p (base[0], base[1], 0))
7330 : return false;
7331 934 : if (maybe_ne (bitsize[0], bitsize[1]))
7332 : return false;
7333 934 : if (offset[0] != offset[1])
7334 : {
7335 916 : if (!offset[0] || !offset[1])
7336 : return false;
7337 916 : if (!operand_equal_p (offset[0], offset[1], 0))
7338 : {
7339 : tree step[2];
7340 0 : for (int i = 0; i < 2; ++i)
7341 : {
7342 0 : step[i] = integer_one_node;
7343 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7344 : {
7345 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7346 0 : if (is_gimple_assign (def_stmt)
7347 0 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
7348 0 : && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
7349 : == INTEGER_CST))
7350 : {
7351 0 : step[i] = gimple_assign_rhs2 (def_stmt);
7352 0 : offset[i] = gimple_assign_rhs1 (def_stmt);
7353 : }
7354 : }
7355 0 : else if (TREE_CODE (offset[i]) == MULT_EXPR)
7356 : {
7357 0 : step[i] = TREE_OPERAND (offset[i], 1);
7358 0 : offset[i] = TREE_OPERAND (offset[i], 0);
7359 : }
7360 0 : tree rhs1 = NULL_TREE;
7361 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7362 : {
7363 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7364 0 : if (gimple_assign_cast_p (def_stmt))
7365 0 : rhs1 = gimple_assign_rhs1 (def_stmt);
7366 : }
7367 0 : else if (CONVERT_EXPR_P (offset[i]))
7368 0 : rhs1 = TREE_OPERAND (offset[i], 0);
7369 0 : if (rhs1
7370 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7371 0 : && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
7372 0 : && (TYPE_PRECISION (TREE_TYPE (offset[i]))
7373 0 : >= TYPE_PRECISION (TREE_TYPE (rhs1))))
7374 0 : offset[i] = rhs1;
7375 : }
7376 0 : if (!operand_equal_p (offset[0], offset[1], 0)
7377 0 : || !operand_equal_p (step[0], step[1], 0))
7378 0 : return false;
7379 : }
7380 : }
7381 : return true;
7382 : }
7383 :
7384 :
7385 : enum scan_store_kind {
7386 : /* Normal permutation. */
7387 : scan_store_kind_perm,
7388 :
7389 : /* Whole vector left shift permutation with zero init. */
7390 : scan_store_kind_lshift_zero,
7391 :
7392 : /* Whole vector left shift permutation and VEC_COND_EXPR. */
7393 : scan_store_kind_lshift_cond
7394 : };
7395 :
7396 : /* Function check_scan_store.
7397 :
7398 : Verify if we can perform the needed permutations or whole vector shifts.
7399 : Return -1 on failure, otherwise exact log2 of vectype's nunits.
7400 : USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7401 : to do at each step. */
7402 :
7403 : static int
7404 1024 : scan_store_can_perm_p (tree vectype, tree init,
7405 : vec<enum scan_store_kind> *use_whole_vector = NULL)
7406 : {
7407 1024 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7408 1024 : unsigned HOST_WIDE_INT nunits;
7409 1024 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7410 : return -1;
7411 1024 : int units_log2 = exact_log2 (nunits);
7412 1024 : if (units_log2 <= 0)
7413 : return -1;
7414 :
7415 : int i;
7416 : enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
7417 4784 : for (i = 0; i <= units_log2; ++i)
7418 : {
7419 3760 : unsigned HOST_WIDE_INT j, k;
7420 3760 : enum scan_store_kind kind = scan_store_kind_perm;
7421 3760 : vec_perm_builder sel (nunits, nunits, 1);
7422 3760 : sel.quick_grow (nunits);
7423 3760 : if (i == units_log2)
7424 : {
7425 9728 : for (j = 0; j < nunits; ++j)
7426 8704 : sel[j] = nunits - 1;
7427 : }
7428 : else
7429 : {
7430 10416 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7431 7680 : sel[j] = j;
7432 26416 : for (k = 0; j < nunits; ++j, ++k)
7433 23680 : sel[j] = nunits + k;
7434 : }
7435 6496 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7436 3760 : if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
7437 : {
7438 0 : if (i == units_log2)
7439 : return -1;
7440 :
7441 0 : if (whole_vector_shift_kind == scan_store_kind_perm)
7442 : {
7443 0 : if (!can_implement_p (vec_shl_optab, vec_mode))
7444 : return -1;
7445 0 : whole_vector_shift_kind = scan_store_kind_lshift_zero;
7446 : /* Whole vector shifts shift in zeros, so if init is all zero
7447 : constant, there is no need to do anything further. */
7448 0 : if ((TREE_CODE (init) != INTEGER_CST
7449 0 : && TREE_CODE (init) != REAL_CST)
7450 0 : || !initializer_zerop (init))
7451 : {
7452 0 : tree masktype = truth_type_for (vectype);
7453 0 : if (!expand_vec_cond_expr_p (vectype, masktype))
7454 : return -1;
7455 : whole_vector_shift_kind = scan_store_kind_lshift_cond;
7456 : }
7457 : }
7458 0 : kind = whole_vector_shift_kind;
7459 : }
7460 3760 : if (use_whole_vector)
7461 : {
7462 1880 : if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
7463 0 : use_whole_vector->safe_grow_cleared (i, true);
7464 5640 : if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
7465 0 : use_whole_vector->safe_push (kind);
7466 : }
7467 3760 : }
7468 :
7469 : return units_log2;
7470 : }
7471 :
7472 :
7473 : /* Function check_scan_store.
7474 :
7475 : Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7476 :
7477 : static bool
7478 1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
7479 : enum vect_def_type rhs_dt, slp_tree slp_node,
7480 : slp_tree mask_node,
7481 : vect_memory_access_type memory_access_type)
7482 : {
7483 1076 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7484 1076 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7485 1076 : tree ref_type;
7486 :
7487 1076 : gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
7488 1076 : if (SLP_TREE_LANES (slp_node) > 1
7489 1076 : || mask_node
7490 1076 : || memory_access_type != VMAT_CONTIGUOUS
7491 1076 : || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
7492 1076 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
7493 1076 : || loop_vinfo == NULL
7494 1076 : || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
7495 1076 : || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
7496 1076 : || STMT_VINFO_GROUPED_ACCESS (stmt_info)
7497 1076 : || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
7498 1076 : || !integer_zerop (DR_INIT (dr_info->dr))
7499 1076 : || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
7500 2152 : || !alias_sets_conflict_p (get_alias_set (vectype),
7501 1076 : get_alias_set (TREE_TYPE (ref_type))))
7502 : {
7503 0 : if (dump_enabled_p ())
7504 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7505 : "unsupported OpenMP scan store.\n");
7506 : return false;
7507 : }
7508 :
7509 : /* We need to pattern match code built by OpenMP lowering and simplified
7510 : by following optimizations into something we can handle.
7511 : #pragma omp simd reduction(inscan,+:r)
7512 : for (...)
7513 : {
7514 : r += something ();
7515 : #pragma omp scan inclusive (r)
7516 : use (r);
7517 : }
7518 : shall have body with:
7519 : // Initialization for input phase, store the reduction initializer:
7520 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7521 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7522 : D.2042[_21] = 0;
7523 : // Actual input phase:
7524 : ...
7525 : r.0_5 = D.2042[_20];
7526 : _6 = _4 + r.0_5;
7527 : D.2042[_20] = _6;
7528 : // Initialization for scan phase:
7529 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7530 : _26 = D.2043[_25];
7531 : _27 = D.2042[_25];
7532 : _28 = _26 + _27;
7533 : D.2043[_25] = _28;
7534 : D.2042[_25] = _28;
7535 : // Actual scan phase:
7536 : ...
7537 : r.1_8 = D.2042[_20];
7538 : ...
7539 : The "omp simd array" variable D.2042 holds the privatized copy used
7540 : inside of the loop and D.2043 is another one that holds copies of
7541 : the current original list item. The separate GOMP_SIMD_LANE ifn
7542 : kinds are there in order to allow optimizing the initializer store
7543 : and combiner sequence, e.g. if it is originally some C++ish user
7544 : defined reduction, but allow the vectorizer to pattern recognize it
7545 : and turn into the appropriate vectorized scan.
7546 :
7547 : For exclusive scan, this is slightly different:
7548 : #pragma omp simd reduction(inscan,+:r)
7549 : for (...)
7550 : {
7551 : use (r);
7552 : #pragma omp scan exclusive (r)
7553 : r += something ();
7554 : }
7555 : shall have body with:
7556 : // Initialization for input phase, store the reduction initializer:
7557 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7558 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7559 : D.2042[_21] = 0;
7560 : // Actual input phase:
7561 : ...
7562 : r.0_5 = D.2042[_20];
7563 : _6 = _4 + r.0_5;
7564 : D.2042[_20] = _6;
7565 : // Initialization for scan phase:
7566 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7567 : _26 = D.2043[_25];
7568 : D.2044[_25] = _26;
7569 : _27 = D.2042[_25];
7570 : _28 = _26 + _27;
7571 : D.2043[_25] = _28;
7572 : // Actual scan phase:
7573 : ...
7574 : r.1_8 = D.2044[_20];
7575 : ... */
7576 :
7577 1076 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
7578 : {
7579 : /* Match the D.2042[_21] = 0; store above. Just require that
7580 : it is a constant or external definition store. */
7581 564 : if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
7582 : {
7583 0 : fail_init:
7584 0 : if (dump_enabled_p ())
7585 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7586 : "unsupported OpenMP scan initializer store.\n");
7587 : return false;
7588 : }
7589 :
7590 564 : if (! loop_vinfo->scan_map)
7591 322 : loop_vinfo->scan_map = new hash_map<tree, tree>;
7592 564 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7593 564 : tree &cached = loop_vinfo->scan_map->get_or_insert (var);
7594 564 : if (cached)
7595 0 : goto fail_init;
7596 564 : cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
7597 :
7598 : /* These stores can be vectorized normally. */
7599 564 : return true;
7600 : }
7601 :
7602 512 : if (rhs_dt != vect_internal_def)
7603 : {
7604 0 : fail:
7605 0 : if (dump_enabled_p ())
7606 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7607 : "unsupported OpenMP scan combiner pattern.\n");
7608 : return false;
7609 : }
7610 :
7611 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7612 512 : tree rhs = gimple_assign_rhs1 (stmt);
7613 512 : if (TREE_CODE (rhs) != SSA_NAME)
7614 0 : goto fail;
7615 :
7616 512 : gimple *other_store_stmt = NULL;
7617 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7618 512 : bool inscan_var_store
7619 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7620 :
7621 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7622 : {
7623 252 : if (!inscan_var_store)
7624 : {
7625 126 : use_operand_p use_p;
7626 126 : imm_use_iterator iter;
7627 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7628 : {
7629 252 : gimple *use_stmt = USE_STMT (use_p);
7630 252 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7631 126 : continue;
7632 126 : if (gimple_bb (use_stmt) != gimple_bb (stmt)
7633 126 : || !is_gimple_assign (use_stmt)
7634 126 : || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
7635 126 : || other_store_stmt
7636 252 : || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
7637 0 : goto fail;
7638 126 : other_store_stmt = use_stmt;
7639 0 : }
7640 126 : if (other_store_stmt == NULL)
7641 0 : goto fail;
7642 126 : rhs = gimple_assign_lhs (other_store_stmt);
7643 126 : if (!single_imm_use (rhs, &use_p, &other_store_stmt))
7644 0 : goto fail;
7645 : }
7646 : }
7647 260 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
7648 : {
7649 260 : use_operand_p use_p;
7650 260 : imm_use_iterator iter;
7651 780 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7652 : {
7653 520 : gimple *use_stmt = USE_STMT (use_p);
7654 520 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7655 260 : continue;
7656 260 : if (other_store_stmt)
7657 0 : goto fail;
7658 260 : other_store_stmt = use_stmt;
7659 260 : }
7660 : }
7661 : else
7662 0 : goto fail;
7663 :
7664 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7665 512 : if (gimple_bb (def_stmt) != gimple_bb (stmt)
7666 512 : || !is_gimple_assign (def_stmt)
7667 1024 : || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
7668 0 : goto fail;
7669 :
7670 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7671 : /* For pointer addition, we should use the normal plus for the vector
7672 : operation. */
7673 512 : switch (code)
7674 : {
7675 0 : case POINTER_PLUS_EXPR:
7676 0 : code = PLUS_EXPR;
7677 0 : break;
7678 0 : case MULT_HIGHPART_EXPR:
7679 0 : goto fail;
7680 : default:
7681 : break;
7682 : }
7683 512 : if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
7684 0 : goto fail;
7685 :
7686 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7687 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7688 512 : if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
7689 0 : goto fail;
7690 :
7691 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7692 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7693 512 : if (gimple_bb (load1_stmt) != gimple_bb (stmt)
7694 512 : || !gimple_assign_load_p (load1_stmt)
7695 512 : || gimple_bb (load2_stmt) != gimple_bb (stmt)
7696 1024 : || !gimple_assign_load_p (load2_stmt))
7697 0 : goto fail;
7698 :
7699 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7700 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7701 512 : if (load1_stmt_info == NULL
7702 512 : || load2_stmt_info == NULL
7703 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
7704 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
7705 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
7706 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7707 0 : goto fail;
7708 :
7709 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
7710 : {
7711 126 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7712 126 : if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
7713 126 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
7714 0 : goto fail;
7715 126 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7716 126 : tree lrhs;
7717 126 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7718 : lrhs = rhs1;
7719 : else
7720 16 : lrhs = rhs2;
7721 126 : use_operand_p use_p;
7722 126 : imm_use_iterator iter;
7723 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
7724 : {
7725 252 : gimple *use_stmt = USE_STMT (use_p);
7726 252 : if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
7727 126 : continue;
7728 126 : if (other_store_stmt)
7729 0 : goto fail;
7730 126 : other_store_stmt = use_stmt;
7731 126 : }
7732 : }
7733 :
7734 512 : if (other_store_stmt == NULL)
7735 0 : goto fail;
7736 512 : if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
7737 512 : || !gimple_store_p (other_store_stmt))
7738 0 : goto fail;
7739 :
7740 512 : stmt_vec_info other_store_stmt_info
7741 512 : = loop_vinfo->lookup_stmt (other_store_stmt);
7742 512 : if (other_store_stmt_info == NULL
7743 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
7744 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7745 0 : goto fail;
7746 :
7747 512 : gimple *stmt1 = stmt;
7748 512 : gimple *stmt2 = other_store_stmt;
7749 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7750 : std::swap (stmt1, stmt2);
7751 512 : if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
7752 : gimple_assign_rhs1 (load2_stmt)))
7753 : {
7754 162 : std::swap (rhs1, rhs2);
7755 162 : std::swap (load1_stmt, load2_stmt);
7756 162 : std::swap (load1_stmt_info, load2_stmt_info);
7757 : }
7758 512 : if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
7759 : gimple_assign_rhs1 (load1_stmt)))
7760 0 : goto fail;
7761 :
7762 512 : tree var3 = NULL_TREE;
7763 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
7764 512 : && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
7765 : gimple_assign_rhs1 (load2_stmt)))
7766 0 : goto fail;
7767 512 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7768 : {
7769 252 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7770 252 : if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
7771 252 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
7772 0 : goto fail;
7773 252 : var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7774 252 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
7775 252 : || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
7776 504 : || lookup_attribute ("omp simd inscan exclusive",
7777 252 : DECL_ATTRIBUTES (var3)))
7778 0 : goto fail;
7779 : }
7780 :
7781 512 : dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
7782 512 : if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
7783 512 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
7784 0 : goto fail;
7785 :
7786 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7787 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
7788 512 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
7789 512 : || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
7790 1024 : || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7791 512 : == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
7792 0 : goto fail;
7793 :
7794 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7795 256 : std::swap (var1, var2);
7796 :
7797 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7798 : {
7799 252 : if (!lookup_attribute ("omp simd inscan exclusive",
7800 252 : DECL_ATTRIBUTES (var1)))
7801 0 : goto fail;
7802 252 : var1 = var3;
7803 : }
7804 :
7805 512 : if (loop_vinfo->scan_map == NULL)
7806 0 : goto fail;
7807 512 : tree *init = loop_vinfo->scan_map->get (var1);
7808 512 : if (init == NULL)
7809 0 : goto fail;
7810 :
7811 : /* The IL is as expected, now check if we can actually vectorize it.
7812 : Inclusive scan:
7813 : _26 = D.2043[_25];
7814 : _27 = D.2042[_25];
7815 : _28 = _26 + _27;
7816 : D.2043[_25] = _28;
7817 : D.2042[_25] = _28;
7818 : should be vectorized as (where _40 is the vectorized rhs
7819 : from the D.2042[_21] = 0; store):
7820 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7821 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7822 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7823 : _33 = _31 + _32;
7824 : // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7825 : _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7826 : _35 = _33 + _34;
7827 : // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7828 : // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7829 : _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7830 : _37 = _35 + _36;
7831 : // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7832 : // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7833 : _38 = _30 + _37;
7834 : _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7835 : MEM <vector(8) int> [(int *)&D.2043] = _39;
7836 : MEM <vector(8) int> [(int *)&D.2042] = _38;
7837 : Exclusive scan:
7838 : _26 = D.2043[_25];
7839 : D.2044[_25] = _26;
7840 : _27 = D.2042[_25];
7841 : _28 = _26 + _27;
7842 : D.2043[_25] = _28;
7843 : should be vectorized as (where _40 is the vectorized rhs
7844 : from the D.2042[_21] = 0; store):
7845 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7846 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7847 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7848 : _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7849 : _34 = _32 + _33;
7850 : // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7851 : // _31[3]+_31[4], ... _31[5]+.._31[6] };
7852 : _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7853 : _36 = _34 + _35;
7854 : // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7855 : // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7856 : _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7857 : _38 = _36 + _37;
7858 : // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7859 : // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7860 : _39 = _30 + _38;
7861 : _50 = _31 + _39;
7862 : _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7863 : MEM <vector(8) int> [(int *)&D.2044] = _39;
7864 : MEM <vector(8) int> [(int *)&D.2042] = _51; */
7865 512 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7866 512 : optab optab = optab_for_tree_code (code, vectype, optab_default);
7867 512 : if (!optab || !can_implement_p (optab, vec_mode))
7868 0 : goto fail;
7869 :
7870 512 : int units_log2 = scan_store_can_perm_p (vectype, *init);
7871 512 : if (units_log2 == -1)
7872 0 : goto fail;
7873 :
7874 : return true;
7875 : }
7876 :
7877 :
7878 : /* Function vectorizable_scan_store.
7879 :
7880 : Helper of vectorizable_score, arguments like on vectorizable_store.
7881 : Handle only the transformation, checking is done in check_scan_store. */
7882 :
7883 : static bool
7884 512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
7885 : slp_tree slp_node, gimple_stmt_iterator *gsi)
7886 : {
7887 512 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7888 512 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7889 512 : tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
7890 512 : tree vectype = SLP_TREE_VECTYPE (slp_node);
7891 :
7892 512 : if (dump_enabled_p ())
7893 492 : dump_printf_loc (MSG_NOTE, vect_location,
7894 : "transform scan store.\n");
7895 :
7896 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7897 512 : tree rhs = gimple_assign_rhs1 (stmt);
7898 512 : gcc_assert (TREE_CODE (rhs) == SSA_NAME);
7899 :
7900 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7901 512 : bool inscan_var_store
7902 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7903 :
7904 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7905 : {
7906 126 : use_operand_p use_p;
7907 126 : imm_use_iterator iter;
7908 126 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7909 : {
7910 126 : gimple *use_stmt = USE_STMT (use_p);
7911 126 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7912 0 : continue;
7913 126 : rhs = gimple_assign_lhs (use_stmt);
7914 126 : break;
7915 126 : }
7916 : }
7917 :
7918 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7919 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7920 512 : if (code == POINTER_PLUS_EXPR)
7921 0 : code = PLUS_EXPR;
7922 512 : gcc_assert (TREE_CODE_LENGTH (code) == binary_op
7923 : && commutative_tree_code (code));
7924 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7925 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7926 512 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
7927 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7928 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7929 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7930 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7931 512 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7932 512 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7933 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7934 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7935 :
7936 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7937 : {
7938 436 : std::swap (rhs1, rhs2);
7939 436 : std::swap (var1, var2);
7940 436 : std::swap (load1_dr_info, load2_dr_info);
7941 : }
7942 :
7943 512 : tree *init = loop_vinfo->scan_map->get (var1);
7944 512 : gcc_assert (init);
7945 :
7946 512 : unsigned HOST_WIDE_INT nunits;
7947 512 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7948 : gcc_unreachable ();
7949 512 : auto_vec<enum scan_store_kind, 16> use_whole_vector;
7950 512 : int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
7951 512 : gcc_assert (units_log2 > 0);
7952 512 : auto_vec<tree, 16> perms;
7953 512 : perms.quick_grow (units_log2 + 1);
7954 512 : tree zero_vec = NULL_TREE, masktype = NULL_TREE;
7955 2904 : for (int i = 0; i <= units_log2; ++i)
7956 : {
7957 1880 : unsigned HOST_WIDE_INT j, k;
7958 1880 : vec_perm_builder sel (nunits, nunits, 1);
7959 1880 : sel.quick_grow (nunits);
7960 1880 : if (i == units_log2)
7961 4864 : for (j = 0; j < nunits; ++j)
7962 4352 : sel[j] = nunits - 1;
7963 : else
7964 : {
7965 5208 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7966 3840 : sel[j] = j;
7967 13208 : for (k = 0; j < nunits; ++j, ++k)
7968 11840 : sel[j] = nunits + k;
7969 : }
7970 3248 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7971 1880 : if (!use_whole_vector.is_empty ()
7972 0 : && use_whole_vector[i] != scan_store_kind_perm)
7973 : {
7974 0 : if (zero_vec == NULL_TREE)
7975 0 : zero_vec = build_zero_cst (vectype);
7976 0 : if (masktype == NULL_TREE
7977 0 : && use_whole_vector[i] == scan_store_kind_lshift_cond)
7978 0 : masktype = truth_type_for (vectype);
7979 0 : perms[i] = vect_gen_perm_mask_any (vectype, indices);
7980 : }
7981 : else
7982 1880 : perms[i] = vect_gen_perm_mask_checked (vectype, indices);
7983 1880 : }
7984 :
7985 512 : tree vec_oprnd1 = NULL_TREE;
7986 512 : tree vec_oprnd2 = NULL_TREE;
7987 512 : tree vec_oprnd3 = NULL_TREE;
7988 512 : tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
7989 512 : tree dataref_offset = build_int_cst (ref_type, 0);
7990 512 : tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
7991 512 : tree ldataref_ptr = NULL_TREE;
7992 512 : tree orig = NULL_TREE;
7993 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7994 126 : ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
7995 : /* The initialization is invariant. */
7996 512 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
7997 512 : auto_vec<tree> vec_oprnds2;
7998 512 : auto_vec<tree> vec_oprnds3;
7999 512 : if (ldataref_ptr == NULL)
8000 : {
8001 : /* We want to lookup the vector operands of the reduction, not those
8002 : of the store - for SLP we have to use the proper SLP node for the
8003 : lookup, which should be the single child of the scan store. */
8004 386 : vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
8005 : rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
8006 : /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
8007 : them in SLP child order. So we have to swap here with logic
8008 : similar to above. */
8009 386 : stmt_vec_info load
8010 386 : = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
8011 386 : (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
8012 386 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
8013 386 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
8014 386 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
8015 820 : for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
8016 494 : std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
8017 : }
8018 : else
8019 126 : vect_get_vec_defs (vinfo, slp_node,
8020 : rhs2, &vec_oprnds3);
8021 1248 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8022 : {
8023 736 : if (ldataref_ptr == NULL)
8024 554 : vec_oprnd2 = vec_oprnds2[j];
8025 736 : vec_oprnd3 = vec_oprnds3[j];
8026 736 : if (j == 0)
8027 : orig = vec_oprnd3;
8028 224 : else if (!inscan_var_store)
8029 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8030 :
8031 736 : if (ldataref_ptr)
8032 : {
8033 182 : vec_oprnd2 = make_ssa_name (vectype);
8034 182 : tree data_ref = fold_build2 (MEM_REF, vectype,
8035 : unshare_expr (ldataref_ptr),
8036 : dataref_offset);
8037 182 : vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
8038 182 : gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
8039 182 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8040 : }
8041 :
8042 736 : tree v = vec_oprnd2;
8043 3068 : for (int i = 0; i < units_log2; ++i)
8044 : {
8045 2332 : tree new_temp = make_ssa_name (vectype);
8046 2332 : gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
8047 : (zero_vec
8048 0 : && (use_whole_vector[i]
8049 0 : != scan_store_kind_perm))
8050 : ? zero_vec : vec_oprnd1, v,
8051 2332 : perms[i]);
8052 2332 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8053 :
8054 2332 : if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
8055 : {
8056 : /* Whole vector shift shifted in zero bits, but if *init
8057 : is not initializer_zerop, we need to replace those elements
8058 : with elements from vec_oprnd1. */
8059 0 : tree_vector_builder vb (masktype, nunits, 1);
8060 0 : for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
8061 0 : vb.quick_push (k < (HOST_WIDE_INT_1U << i)
8062 : ? boolean_false_node : boolean_true_node);
8063 :
8064 0 : tree new_temp2 = make_ssa_name (vectype);
8065 0 : g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
8066 : new_temp, vec_oprnd1);
8067 0 : vect_finish_stmt_generation (vinfo, stmt_info,
8068 : g, gsi);
8069 0 : new_temp = new_temp2;
8070 0 : }
8071 :
8072 : /* For exclusive scan, perform the perms[i] permutation once
8073 : more. */
8074 2332 : if (i == 0
8075 1100 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
8076 728 : && v == vec_oprnd2)
8077 : {
8078 364 : v = new_temp;
8079 364 : --i;
8080 364 : continue;
8081 : }
8082 :
8083 1968 : tree new_temp2 = make_ssa_name (vectype);
8084 1968 : g = gimple_build_assign (new_temp2, code, v, new_temp);
8085 1968 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8086 :
8087 1968 : v = new_temp2;
8088 : }
8089 :
8090 736 : tree new_temp = make_ssa_name (vectype);
8091 736 : gimple *g = gimple_build_assign (new_temp, code, orig, v);
8092 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8093 :
8094 736 : tree last_perm_arg = new_temp;
8095 : /* For exclusive scan, new_temp computed above is the exclusive scan
8096 : prefix sum. Turn it into inclusive prefix sum for the broadcast
8097 : of the last element into orig. */
8098 736 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
8099 : {
8100 364 : last_perm_arg = make_ssa_name (vectype);
8101 364 : g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
8102 364 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8103 : }
8104 :
8105 736 : orig = make_ssa_name (vectype);
8106 2208 : g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
8107 736 : last_perm_arg, perms[units_log2]);
8108 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8109 :
8110 736 : if (!inscan_var_store)
8111 : {
8112 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8113 : unshare_expr (dataref_ptr),
8114 : dataref_offset);
8115 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8116 368 : g = gimple_build_assign (data_ref, new_temp);
8117 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8118 : }
8119 : }
8120 :
8121 512 : if (inscan_var_store)
8122 624 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8123 : {
8124 368 : if (j != 0)
8125 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8126 :
8127 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8128 : unshare_expr (dataref_ptr),
8129 : dataref_offset);
8130 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8131 368 : gimple *g = gimple_build_assign (data_ref, orig);
8132 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8133 : }
8134 512 : return true;
8135 512 : }
8136 :
8137 :
8138 : /* Function vectorizable_store.
8139 :
8140 : Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8141 : that can be vectorized.
8142 : If COST_VEC is passed, calculate costs but don't change anything,
8143 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
8144 : it, and insert it at GSI.
8145 : Return true if STMT_INFO is vectorizable in this way. */
8146 :
8147 : static bool
8148 2154442 : vectorizable_store (vec_info *vinfo,
8149 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8150 : slp_tree slp_node,
8151 : stmt_vector_for_cost *cost_vec)
8152 : {
8153 2154442 : tree data_ref;
8154 2154442 : tree vec_oprnd = NULL_TREE;
8155 2154442 : tree elem_type;
8156 2154442 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
8157 2154442 : class loop *loop = NULL;
8158 2154442 : machine_mode vec_mode;
8159 2154442 : tree dummy;
8160 2154442 : enum vect_def_type rhs_dt = vect_unknown_def_type;
8161 2154442 : enum vect_def_type mask_dt = vect_unknown_def_type;
8162 2154442 : tree dataref_ptr = NULL_TREE;
8163 2154442 : tree dataref_offset = NULL_TREE;
8164 2154442 : int j;
8165 2154442 : stmt_vec_info first_stmt_info;
8166 2154442 : bool grouped_store;
8167 2154442 : unsigned int group_size, i;
8168 2154442 : unsigned int vec_num;
8169 2154442 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
8170 2154442 : tree aggr_type;
8171 2154442 : poly_uint64 vf;
8172 2154442 : vec_load_store_type vls_type;
8173 2154442 : tree ref_type;
8174 :
8175 2154442 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8176 : return false;
8177 :
8178 2154442 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8179 247311 : && cost_vec)
8180 : return false;
8181 :
8182 : /* Is vectorizable store? */
8183 :
8184 1907131 : tree mask_vectype = NULL_TREE;
8185 1907131 : slp_tree mask_node = NULL;
8186 1907131 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
8187 : {
8188 1832139 : tree scalar_dest = gimple_assign_lhs (assign);
8189 1832139 : if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
8190 1832139 : && is_pattern_stmt_p (stmt_info))
8191 1677 : scalar_dest = TREE_OPERAND (scalar_dest, 0);
8192 1832139 : if (TREE_CODE (scalar_dest) != ARRAY_REF
8193 1832139 : && TREE_CODE (scalar_dest) != BIT_FIELD_REF
8194 : && TREE_CODE (scalar_dest) != INDIRECT_REF
8195 : && TREE_CODE (scalar_dest) != COMPONENT_REF
8196 : && TREE_CODE (scalar_dest) != IMAGPART_EXPR
8197 : && TREE_CODE (scalar_dest) != REALPART_EXPR
8198 : && TREE_CODE (scalar_dest) != MEM_REF)
8199 : return false;
8200 : }
8201 : else
8202 : {
8203 74992 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
8204 13272 : if (!call || !gimple_call_internal_p (call))
8205 : return false;
8206 :
8207 8083 : internal_fn ifn = gimple_call_internal_fn (call);
8208 8083 : if (!internal_store_fn_p (ifn))
8209 : return false;
8210 :
8211 1857 : int mask_index = internal_fn_mask_index (ifn);
8212 1857 : if (mask_index >= 0)
8213 1857 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
8214 1857 : if (mask_index >= 0
8215 1857 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
8216 : &mask_node, &mask_dt,
8217 : &mask_vectype))
8218 : return false;
8219 : }
8220 :
8221 1385425 : tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
8222 1385425 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8223 :
8224 1385425 : if (loop_vinfo)
8225 : {
8226 232502 : loop = LOOP_VINFO_LOOP (loop_vinfo);
8227 232502 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8228 : }
8229 : else
8230 : vf = 1;
8231 1385425 : vec_num = vect_get_num_copies (vinfo, slp_node);
8232 :
8233 : /* FORNOW. This restriction should be relaxed. */
8234 1385425 : if (loop
8235 1385712 : && nested_in_vect_loop_p (loop, stmt_info)
8236 1385720 : && vec_num > 1)
8237 : {
8238 8 : if (dump_enabled_p ())
8239 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8240 : "multiple types in nested loop.\n");
8241 : return false;
8242 : }
8243 :
8244 1385417 : slp_tree op_node;
8245 1385417 : if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
8246 : &op_node, &rhs_dt, &rhs_vectype, &vls_type))
8247 : return false;
8248 :
8249 1385393 : elem_type = TREE_TYPE (vectype);
8250 1385393 : vec_mode = TYPE_MODE (vectype);
8251 :
8252 1385393 : if (!STMT_VINFO_DATA_REF (stmt_info))
8253 : return false;
8254 :
8255 1385393 : vect_load_store_data _ls_data{};
8256 1385393 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
8257 1385393 : if (cost_vec
8258 1385393 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
8259 : vls_type, &_ls_data))
8260 : return false;
8261 : /* Temporary aliases to analysis data, should not be modified through
8262 : these. */
8263 1384769 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
8264 1384769 : const dr_alignment_support alignment_support_scheme
8265 : = ls.alignment_support_scheme;
8266 1384769 : const int misalignment = ls.misalignment;
8267 1384769 : const poly_int64 poffset = ls.poffset;
8268 :
8269 1384769 : if (slp_node->ldst_lanes
8270 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
8271 : {
8272 0 : if (dump_enabled_p ())
8273 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8274 : "discovered store-lane but cannot use it.\n");
8275 : return false;
8276 : }
8277 :
8278 1384769 : if (mask_node)
8279 : {
8280 1767 : if (memory_access_type == VMAT_CONTIGUOUS)
8281 : {
8282 616 : if (!VECTOR_MODE_P (vec_mode)
8283 3018 : || !can_vec_mask_load_store_p (vec_mode,
8284 1509 : TYPE_MODE (mask_vectype), false))
8285 : return false;
8286 : }
8287 258 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
8288 258 : && (!mat_gather_scatter_p (memory_access_type)
8289 242 : || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
8290 170 : && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
8291 : {
8292 16 : if (dump_enabled_p ())
8293 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8294 : "unsupported access type for masked store.\n");
8295 : return false;
8296 : }
8297 242 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
8298 : {
8299 72 : if (dump_enabled_p ())
8300 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8301 : "unsupported masked emulated scatter.\n");
8302 : return false;
8303 : }
8304 : }
8305 : else
8306 : {
8307 : /* FORNOW. In some cases can vectorize even if data-type not supported
8308 : (e.g. - array initialization with 0). */
8309 1383002 : if (!can_implement_p (mov_optab, vec_mode))
8310 : return false;
8311 : }
8312 :
8313 1384567 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
8314 1384567 : grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
8315 2557749 : && !mat_gather_scatter_p (memory_access_type));
8316 1173182 : if (grouped_store)
8317 : {
8318 1173182 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8319 1173182 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8320 1173182 : group_size = DR_GROUP_SIZE (first_stmt_info);
8321 : }
8322 : else
8323 : {
8324 1384567 : first_stmt_info = stmt_info;
8325 1384567 : first_dr_info = dr_info;
8326 : group_size = 1;
8327 : }
8328 :
8329 1384567 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
8330 : {
8331 1076 : if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
8332 : mask_node, memory_access_type))
8333 : return false;
8334 : }
8335 :
8336 1934078 : bool costing_p = cost_vec;
8337 1383799 : if (costing_p) /* transformation not required. */
8338 : {
8339 834288 : if (loop_vinfo
8340 166233 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
8341 77877 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
8342 : vls_type, group_size, &ls,
8343 : mask_node);
8344 :
8345 834288 : if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
8346 834288 : || (mask_node
8347 1038 : && !vect_maybe_update_slp_op_vectype (mask_node,
8348 : mask_vectype)))
8349 : {
8350 0 : if (dump_enabled_p ())
8351 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8352 : "incompatible vector types for invariants\n");
8353 : return false;
8354 : }
8355 :
8356 834288 : if (dump_enabled_p ()
8357 : && memory_access_type != VMAT_ELEMENTWISE
8358 15663 : && memory_access_type != VMAT_STRIDED_SLP
8359 14999 : && memory_access_type != VMAT_INVARIANT
8360 849287 : && alignment_support_scheme != dr_aligned)
8361 5235 : dump_printf_loc (MSG_NOTE, vect_location,
8362 : "Vectorizing an unaligned access.\n");
8363 : }
8364 :
8365 : /* Transform. */
8366 :
8367 : if (!costing_p)
8368 550279 : ensure_base_align (dr_info);
8369 :
8370 1384567 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
8371 : {
8372 1024 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
8373 1024 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8374 1024 : if (costing_p)
8375 : {
8376 512 : unsigned int inside_cost = 0, prologue_cost = 0;
8377 512 : if (vls_type == VLS_STORE_INVARIANT)
8378 0 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8379 : slp_node, 0, vect_prologue);
8380 512 : vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
8381 : alignment_support_scheme, misalignment,
8382 : &inside_cost, cost_vec);
8383 :
8384 512 : if (dump_enabled_p ())
8385 492 : dump_printf_loc (MSG_NOTE, vect_location,
8386 : "vect_model_store_cost: inside_cost = %d, "
8387 : "prologue_cost = %d .\n",
8388 : inside_cost, prologue_cost);
8389 :
8390 512 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8391 512 : slp_node->data = new vect_load_store_data (std::move (ls));
8392 :
8393 512 : return true;
8394 : }
8395 512 : return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
8396 : }
8397 :
8398 : /* FORNOW */
8399 1383543 : gcc_assert (!grouped_store
8400 : || !loop
8401 : || !nested_in_vect_loop_p (loop, stmt_info));
8402 :
8403 1383543 : grouped_store = false;
8404 1383543 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
8405 1383543 : gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
8406 : || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
8407 1383543 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8408 :
8409 1383543 : ref_type = get_group_alias_ptr_type (first_stmt_info);
8410 :
8411 1383543 : if (!costing_p && dump_enabled_p ())
8412 12678 : dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
8413 :
8414 1383543 : if (memory_access_type == VMAT_ELEMENTWISE
8415 1383543 : || memory_access_type == VMAT_STRIDED_SLP)
8416 : {
8417 29515 : unsigned inside_cost = 0, prologue_cost = 0;
8418 29515 : gimple_stmt_iterator incr_gsi;
8419 29515 : bool insert_after;
8420 29515 : tree offvar = NULL_TREE;
8421 29515 : tree ivstep;
8422 29515 : tree running_off;
8423 29515 : tree stride_base, stride_step, alias_off;
8424 29515 : tree vec_oprnd = NULL_TREE;
8425 29515 : tree dr_offset;
8426 : /* Checked by get_load_store_type. */
8427 29515 : unsigned int const_nunits = nunits.to_constant ();
8428 :
8429 29515 : gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
8430 29515 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
8431 :
8432 29515 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
8433 29515 : stride_base
8434 29515 : = fold_build_pointer_plus
8435 : (DR_BASE_ADDRESS (first_dr_info->dr),
8436 : size_binop (PLUS_EXPR,
8437 : convert_to_ptrofftype (dr_offset),
8438 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
8439 29515 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
8440 :
8441 : /* For a store with loop-invariant (but other than power-of-2)
8442 : stride (i.e. not a grouped access) like so:
8443 :
8444 : for (i = 0; i < n; i += stride)
8445 : array[i] = ...;
8446 :
8447 : we generate a new induction variable and new stores from
8448 : the components of the (vectorized) rhs:
8449 :
8450 : for (j = 0; ; j += VF*stride)
8451 : vectemp = ...;
8452 : tmp1 = vectemp[0];
8453 : array[j] = tmp1;
8454 : tmp2 = vectemp[1];
8455 : array[j + stride] = tmp2;
8456 : ...
8457 : */
8458 :
8459 : /* ??? Modify local copies of alignment_support_scheme and
8460 : misalignment, but this part of analysis should be done
8461 : earlier and remembered, likewise the chosen load mode. */
8462 29515 : const dr_alignment_support tem = alignment_support_scheme;
8463 29515 : dr_alignment_support alignment_support_scheme = tem;
8464 29515 : const int tem2 = misalignment;
8465 29515 : int misalignment = tem2;
8466 :
8467 29515 : unsigned nstores = const_nunits;
8468 29515 : unsigned lnel = 1;
8469 29515 : tree ltype = elem_type;
8470 29515 : tree lvectype = vectype;
8471 29515 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
8472 29515 : if (n == const_nunits)
8473 : {
8474 2980 : int mis_align = dr_misalignment (first_dr_info, vectype);
8475 : /* With VF > 1 we advance the DR by step, if that is constant
8476 : and only aligned when performed VF times, DR alignment
8477 : analysis can analyze this as aligned since it assumes
8478 : contiguous accesses. But that is not how we code generate
8479 : here, so adjust for this. */
8480 2980 : if (maybe_gt (vf, 1u)
8481 4518 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8482 4283 : DR_TARGET_ALIGNMENT (first_dr_info)))
8483 235 : mis_align = -1;
8484 2980 : dr_alignment_support dr_align
8485 2980 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
8486 : mis_align);
8487 2980 : if (dr_align == dr_aligned
8488 2980 : || dr_align == dr_unaligned_supported)
8489 : {
8490 29515 : nstores = 1;
8491 29515 : lnel = const_nunits;
8492 29515 : ltype = vectype;
8493 29515 : lvectype = vectype;
8494 29515 : alignment_support_scheme = dr_align;
8495 29515 : misalignment = mis_align;
8496 : }
8497 : }
8498 26535 : else if (n > 1)
8499 : {
8500 1969 : nstores = const_nunits / n;
8501 1969 : lnel = n;
8502 1969 : ltype = build_vector_type (elem_type, n);
8503 1969 : lvectype = vectype;
8504 1969 : int mis_align = dr_misalignment (first_dr_info, ltype);
8505 1969 : if (maybe_gt (vf, 1u)
8506 3938 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8507 3292 : DR_TARGET_ALIGNMENT (first_dr_info)))
8508 646 : mis_align = -1;
8509 1969 : dr_alignment_support dr_align
8510 1969 : = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
8511 : mis_align);
8512 1969 : alignment_support_scheme = dr_align;
8513 1969 : misalignment = mis_align;
8514 :
8515 : /* First check if vec_extract optab doesn't support extraction
8516 : of vector elts directly. */
8517 1969 : scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
8518 1969 : machine_mode vmode;
8519 3938 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8520 3744 : || !related_vector_mode (TYPE_MODE (vectype), elmode,
8521 2163 : n).exists (&vmode)
8522 1775 : || (convert_optab_handler (vec_extract_optab,
8523 1775 : TYPE_MODE (vectype), vmode)
8524 : == CODE_FOR_nothing)
8525 1969 : || !(dr_align == dr_aligned
8526 172 : || dr_align == dr_unaligned_supported))
8527 : {
8528 : /* Try to avoid emitting an extract of vector elements
8529 : by performing the extracts using an integer type of the
8530 : same size, extracting from a vector of those and then
8531 : re-interpreting it as the original vector type if
8532 : supported. */
8533 1797 : unsigned lsize = n * GET_MODE_BITSIZE (elmode);
8534 1797 : unsigned int lnunits = const_nunits / n;
8535 : /* If we can't construct such a vector fall back to
8536 : element extracts from the original vector type and
8537 : element size stores. */
8538 1797 : if (int_mode_for_size (lsize, 0).exists (&elmode)
8539 1797 : && VECTOR_MODE_P (TYPE_MODE (vectype))
8540 1797 : && related_vector_mode (TYPE_MODE (vectype), elmode,
8541 1797 : lnunits).exists (&vmode)
8542 1769 : && (convert_optab_handler (vec_extract_optab,
8543 : vmode, elmode)
8544 : != CODE_FOR_nothing))
8545 : {
8546 1769 : nstores = lnunits;
8547 1769 : lnel = n;
8548 1769 : ltype = build_nonstandard_integer_type (lsize, 1);
8549 1769 : lvectype = build_vector_type (ltype, nstores);
8550 : }
8551 : /* Else fall back to vector extraction anyway.
8552 : Fewer stores are more important than avoiding spilling
8553 : of the vector we extract from. Compared to the
8554 : construction case in vectorizable_load no store-forwarding
8555 : issue exists here for reasonable archs. But only
8556 : if the store is supported. */
8557 28 : else if (!(dr_align == dr_aligned
8558 28 : || dr_align == dr_unaligned_supported))
8559 : {
8560 29515 : nstores = const_nunits;
8561 29515 : lnel = 1;
8562 29515 : ltype = elem_type;
8563 29515 : lvectype = vectype;
8564 : }
8565 : }
8566 : }
8567 :
8568 29515 : if (costing_p)
8569 : {
8570 : /* Record the decomposition type for target access during costing. */
8571 26061 : ls.ls_type = lvectype;
8572 26061 : ls.ls_eltype = ltype;
8573 : }
8574 : else
8575 3454 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
8576 :
8577 29515 : unsigned align;
8578 29515 : if (alignment_support_scheme == dr_aligned)
8579 1263 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
8580 : else
8581 28252 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
8582 : /* Alignment is at most the access size if we do multiple stores. */
8583 29515 : if (nstores > 1)
8584 26535 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
8585 29515 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
8586 29515 : int ncopies = vec_num;
8587 :
8588 29515 : if (!costing_p)
8589 : {
8590 3454 : ivstep = stride_step;
8591 :
8592 3454 : tree increment = fold_convert (TREE_TYPE (ivstep),
8593 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
8594 :
8595 3454 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
8596 : increment);
8597 :
8598 3454 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
8599 :
8600 3454 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
8601 3454 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
8602 3454 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
8603 : else
8604 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
8605 : true, NULL_TREE, true,
8606 : GSI_SAME_STMT);
8607 :
8608 3454 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
8609 : insert_after, &offvar, NULL,
8610 3454 : LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
8611 :
8612 3454 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
8613 : }
8614 :
8615 29515 : alias_off = build_int_cst (ref_type, 0);
8616 29515 : auto_vec<tree> vec_oprnds;
8617 : /* For costing some adjacent vector stores, we'd like to cost with
8618 : the total number of them once instead of cost each one by one. */
8619 29515 : unsigned int n_adjacent_stores = 0;
8620 29515 : running_off = offvar;
8621 29515 : if (!costing_p)
8622 3454 : vect_get_slp_defs (op_node, &vec_oprnds);
8623 29515 : unsigned int group_el = 0;
8624 29515 : unsigned HOST_WIDE_INT elsz
8625 29515 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
8626 70347 : for (j = 0; j < ncopies; j++)
8627 : {
8628 40832 : if (!costing_p)
8629 : {
8630 5350 : vec_oprnd = vec_oprnds[j];
8631 : /* Pun the vector to extract from if necessary. */
8632 5350 : if (lvectype != vectype)
8633 : {
8634 1010 : tree tem = make_ssa_name (lvectype);
8635 1010 : tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
8636 1010 : gimple *pun = gimple_build_assign (tem, cvt);
8637 1010 : vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
8638 1010 : vec_oprnd = tem;
8639 : }
8640 : }
8641 181697 : for (i = 0; i < nstores; i++)
8642 : {
8643 140865 : if (costing_p)
8644 : {
8645 124723 : n_adjacent_stores++;
8646 124723 : continue;
8647 : }
8648 16142 : tree newref, newoff;
8649 16142 : gimple *incr, *assign;
8650 16142 : tree size = TYPE_SIZE (ltype);
8651 : /* Extract the i'th component. */
8652 16142 : tree pos = fold_build2 (MULT_EXPR, bitsizetype,
8653 : bitsize_int (i), size);
8654 16142 : tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
8655 : size, pos);
8656 :
8657 16142 : elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
8658 : GSI_SAME_STMT);
8659 :
8660 16142 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
8661 16142 : group_el * elsz);
8662 16142 : newref = build2 (MEM_REF, ltype, running_off, this_off);
8663 16142 : vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
8664 :
8665 : /* And store it to *running_off. */
8666 16142 : assign = gimple_build_assign (newref, elem);
8667 16142 : vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
8668 :
8669 16142 : group_el += lnel;
8670 16142 : if (group_el == group_size)
8671 : {
8672 14466 : newoff = copy_ssa_name (running_off, NULL);
8673 14466 : incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
8674 : running_off, stride_step);
8675 14466 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
8676 :
8677 14466 : running_off = newoff;
8678 14466 : group_el = 0;
8679 : }
8680 : }
8681 : }
8682 :
8683 29515 : if (costing_p)
8684 : {
8685 26061 : if (n_adjacent_stores > 0)
8686 : {
8687 : /* Take a single lane vector type store as scalar
8688 : store to avoid ICE like 110776. */
8689 26061 : if (VECTOR_TYPE_P (ltype)
8690 26061 : && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
8691 1616 : vect_get_store_cost (vinfo, stmt_info, slp_node,
8692 : n_adjacent_stores, alignment_support_scheme,
8693 : misalignment, &inside_cost, cost_vec);
8694 : else
8695 24445 : inside_cost
8696 24445 : += record_stmt_cost (cost_vec, n_adjacent_stores,
8697 : scalar_store, slp_node, 0, vect_body);
8698 : /* Only need vector deconstruction when there is more
8699 : than one store. */
8700 26061 : if (nstores > 1)
8701 24029 : inside_cost
8702 24029 : += record_stmt_cost (cost_vec, ncopies,
8703 : vec_deconstruct, slp_node, 0, vect_body);
8704 : }
8705 26061 : if (dump_enabled_p ())
8706 664 : dump_printf_loc (MSG_NOTE, vect_location,
8707 : "vect_model_store_cost: inside_cost = %d, "
8708 : "prologue_cost = %d .\n",
8709 : inside_cost, prologue_cost);
8710 :
8711 26061 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8712 26061 : slp_node->data = new vect_load_store_data (std::move (ls));
8713 : }
8714 :
8715 29515 : return true;
8716 29515 : }
8717 :
8718 1354028 : gcc_assert (alignment_support_scheme);
8719 1354028 : vec_loop_masks *loop_masks
8720 201105 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
8721 1354028 : ? &LOOP_VINFO_MASKS (loop_vinfo)
8722 12 : : NULL);
8723 12 : vec_loop_lens *loop_lens
8724 201105 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
8725 : ? &LOOP_VINFO_LENS (loop_vinfo)
8726 0 : : NULL);
8727 :
8728 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8729 : are some difference here. We cannot enable both the lens and masks
8730 : during transform but it is allowed during analysis.
8731 : Shouldn't go with length-based approach if fully masked. */
8732 1354028 : if (cost_vec == NULL)
8733 : /* The cost_vec is NULL during transform. */
8734 546313 : gcc_assert ((!loop_lens || !loop_masks));
8735 :
8736 : /* Targets with store-lane instructions must not require explicit
8737 : realignment. vect_supportable_dr_alignment always returns either
8738 : dr_aligned or dr_unaligned_supported for masked operations. */
8739 1354028 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
8740 : && !mask_node
8741 : && !loop_masks)
8742 : || alignment_support_scheme == dr_aligned
8743 : || alignment_support_scheme == dr_unaligned_supported);
8744 :
8745 1354028 : tree offset = NULL_TREE;
8746 1354028 : if (!known_eq (poffset, 0))
8747 4675 : offset = size_int (poffset);
8748 :
8749 1354028 : tree dr_increment;
8750 1354028 : tree dr_bump;
8751 :
8752 1354028 : tree vec_offset = NULL_TREE;
8753 1354028 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8754 : {
8755 1465 : aggr_type = NULL_TREE;
8756 1465 : dr_increment = NULL_TREE;
8757 1465 : dr_bump = NULL_TREE;
8758 : }
8759 1352563 : else if (mat_gather_scatter_p (memory_access_type))
8760 : {
8761 0 : aggr_type = elem_type;
8762 0 : if (!costing_p)
8763 : {
8764 0 : tree vtype = ls.ls_type ? ls.ls_type : vectype;
8765 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
8766 : ls.strided_offset_vectype,
8767 : loop_vinfo, gsi,
8768 : &dr_increment, &dr_bump,
8769 : &vec_offset);
8770 : }
8771 : }
8772 : else
8773 : {
8774 1352563 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8775 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
8776 : else
8777 : aggr_type = vectype;
8778 1352563 : if (!costing_p)
8779 : {
8780 545833 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
8781 : memory_access_type);
8782 545833 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
8783 : memory_access_type);
8784 : }
8785 : }
8786 :
8787 1354028 : if (loop_vinfo && mask_node && !costing_p)
8788 527 : LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
8789 :
8790 : /* In case the vectorization factor (VF) is bigger than the number
8791 : of elements that we can fit in a vectype (nunits), we have to generate
8792 : more than one vector stmt - i.e - we need to "unroll" the
8793 : vector stmt by a factor VF/nunits. */
8794 :
8795 1354028 : auto_vec<tree> dr_chain (group_size);
8796 1354028 : auto_vec<tree> vec_masks;
8797 1354028 : tree vec_mask = NULL;
8798 1354028 : auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
8799 6101504 : for (i = 0; i < group_size; i++)
8800 3393448 : gvec_oprnds.quick_push (new auto_vec<tree> ());
8801 :
8802 1354028 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8803 : {
8804 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
8805 :
8806 0 : if (costing_p)
8807 : /* Update all incoming store operand nodes, the general handling
8808 : above only handles the mask and the first store operand node. */
8809 0 : for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
8810 0 : if (child != mask_node
8811 0 : && !vect_maybe_update_slp_op_vectype (child, vectype))
8812 : {
8813 0 : if (dump_enabled_p ())
8814 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8815 : "incompatible vector types for invariants\n");
8816 : return false;
8817 : }
8818 0 : unsigned inside_cost = 0, prologue_cost = 0;
8819 : /* For costing some adjacent vector stores, we'd like to cost with
8820 : the total number of them once instead of cost each one by one. */
8821 0 : unsigned int n_adjacent_stores = 0;
8822 0 : int ncopies = vec_num / group_size;
8823 0 : for (j = 0; j < ncopies; j++)
8824 : {
8825 0 : if (j == 0)
8826 : {
8827 0 : if (!costing_p)
8828 : {
8829 0 : if (mask_node)
8830 : {
8831 0 : vect_get_slp_defs (mask_node, &vec_masks);
8832 0 : vec_mask = vec_masks[0];
8833 : }
8834 0 : dataref_ptr
8835 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8836 : aggr_type, NULL, offset, &dummy,
8837 : gsi, NULL, false, dr_increment);
8838 : }
8839 : }
8840 0 : else if (!costing_p)
8841 : {
8842 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8843 0 : if (mask_node)
8844 0 : vec_mask = vec_masks[j];
8845 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
8846 : stmt_info, dr_bump);
8847 : }
8848 :
8849 0 : if (costing_p)
8850 : {
8851 0 : n_adjacent_stores += group_size;
8852 0 : continue;
8853 : }
8854 :
8855 : /* Get an array into which we can store the individual vectors. */
8856 0 : tree vec_array = create_vector_array (vectype, group_size);
8857 :
8858 : /* Invalidate the current contents of VEC_ARRAY. This should
8859 : become an RTL clobber too, which prevents the vector registers
8860 : from being upward-exposed. */
8861 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8862 :
8863 : /* Store the individual vectors into the array. */
8864 0 : for (i = 0; i < group_size; i++)
8865 : {
8866 0 : slp_tree child;
8867 0 : if (i == 0 || !mask_node)
8868 0 : child = SLP_TREE_CHILDREN (slp_node)[i];
8869 : else
8870 0 : child = SLP_TREE_CHILDREN (slp_node)[i + 1];
8871 0 : vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
8872 0 : write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
8873 : i);
8874 : }
8875 :
8876 0 : tree final_mask = NULL;
8877 0 : tree final_len = NULL;
8878 0 : tree bias = NULL;
8879 0 : if (loop_masks)
8880 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
8881 : ncopies, vectype, j);
8882 0 : if (vec_mask)
8883 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
8884 : vec_mask, gsi);
8885 :
8886 0 : if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
8887 : {
8888 0 : if (loop_lens)
8889 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
8890 : ncopies, vectype, j, 1, true);
8891 : else
8892 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8893 0 : signed char biasval
8894 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
8895 0 : bias = build_int_cst (intQI_type_node, biasval);
8896 0 : if (!final_mask)
8897 : {
8898 0 : mask_vectype = truth_type_for (vectype);
8899 0 : final_mask = build_minus_one_cst (mask_vectype);
8900 : }
8901 : }
8902 :
8903 0 : gcall *call;
8904 0 : if (final_len && final_mask)
8905 : {
8906 : /* Emit:
8907 : MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8908 : LEN, BIAS, VEC_ARRAY). */
8909 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8910 0 : tree alias_ptr = build_int_cst (ref_type, align);
8911 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
8912 : dataref_ptr, alias_ptr,
8913 : final_mask, final_len, bias,
8914 : vec_array);
8915 : }
8916 0 : else if (final_mask)
8917 : {
8918 : /* Emit:
8919 : MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8920 : VEC_ARRAY). */
8921 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8922 0 : tree alias_ptr = build_int_cst (ref_type, align);
8923 0 : call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
8924 : dataref_ptr, alias_ptr,
8925 : final_mask, vec_array);
8926 : }
8927 : else
8928 : {
8929 : /* Emit:
8930 : MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8931 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8932 0 : call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
8933 0 : gimple_call_set_lhs (call, data_ref);
8934 : }
8935 0 : gimple_call_set_nothrow (call, true);
8936 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
8937 :
8938 : /* Record that VEC_ARRAY is now dead. */
8939 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8940 : }
8941 :
8942 0 : if (costing_p)
8943 : {
8944 0 : if (n_adjacent_stores > 0)
8945 0 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
8946 : alignment_support_scheme, misalignment,
8947 : &inside_cost, cost_vec);
8948 0 : if (dump_enabled_p ())
8949 0 : dump_printf_loc (MSG_NOTE, vect_location,
8950 : "vect_model_store_cost: inside_cost = %d, "
8951 : "prologue_cost = %d .\n",
8952 : inside_cost, prologue_cost);
8953 :
8954 0 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8955 0 : slp_node->data = new vect_load_store_data (std::move (ls));
8956 : }
8957 :
8958 : return true;
8959 : }
8960 :
8961 1354028 : if (mat_gather_scatter_p (memory_access_type))
8962 : {
8963 1465 : gcc_assert (!grouped_store || ls.ls_type);
8964 1465 : if (ls.ls_type)
8965 0 : vectype = ls.ls_type;
8966 1465 : auto_vec<tree> vec_offsets;
8967 1465 : unsigned int inside_cost = 0, prologue_cost = 0;
8968 1465 : int num_stmts = vec_num;
8969 3329 : for (j = 0; j < num_stmts; j++)
8970 : {
8971 1864 : gimple *new_stmt;
8972 1864 : if (j == 0)
8973 : {
8974 1465 : if (costing_p && vls_type == VLS_STORE_INVARIANT)
8975 202 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8976 : slp_node, 0, vect_prologue);
8977 : else if (!costing_p)
8978 : {
8979 : /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8980 : DR_CHAIN is of size 1. */
8981 480 : gcc_assert (group_size == 1);
8982 480 : vect_get_slp_defs (op_node, gvec_oprnds[0]);
8983 480 : if (mask_node)
8984 70 : vect_get_slp_defs (mask_node, &vec_masks);
8985 :
8986 480 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8987 480 : vect_get_gather_scatter_ops (loop, slp_node,
8988 : &dataref_ptr, &vec_offsets);
8989 : else
8990 0 : dataref_ptr
8991 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8992 : aggr_type, NULL, offset,
8993 : &dummy, gsi, NULL, false,
8994 : dr_increment);
8995 : }
8996 : }
8997 399 : else if (!costing_p)
8998 : {
8999 35 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
9000 35 : if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9001 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
9002 : gsi, stmt_info, dr_bump);
9003 : }
9004 :
9005 2581 : new_stmt = NULL;
9006 717 : if (!costing_p)
9007 : {
9008 515 : vec_oprnd = (*gvec_oprnds[0])[j];
9009 515 : if (mask_node)
9010 90 : vec_mask = vec_masks[j];
9011 : /* We should have caught mismatched types earlier. */
9012 515 : gcc_assert (ls.ls_type
9013 : || useless_type_conversion_p
9014 : (vectype, TREE_TYPE (vec_oprnd)));
9015 : }
9016 515 : tree final_mask = NULL_TREE;
9017 2379 : tree final_len = NULL_TREE;
9018 2379 : tree bias = NULL_TREE;
9019 515 : if (!costing_p)
9020 : {
9021 515 : if (loop_masks)
9022 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi,
9023 : loop_masks, num_stmts,
9024 : vectype, j);
9025 515 : if (vec_mask)
9026 90 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
9027 : final_mask, vec_mask, gsi);
9028 : }
9029 :
9030 1864 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
9031 1864 : tree alias_align_ptr = build_int_cst (ref_type, align);
9032 1864 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
9033 : {
9034 0 : if (costing_p)
9035 : {
9036 0 : if (ls.supported_offset_vectype
9037 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
9038 : vec_offset))
9039 0 : inside_cost
9040 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9041 : slp_node, 0, vect_body);
9042 0 : if (ls.supported_scale)
9043 0 : inside_cost
9044 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9045 : slp_node, 0, vect_body);
9046 :
9047 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9048 0 : inside_cost
9049 0 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9050 : slp_node, 0, vect_body);
9051 0 : continue;
9052 0 : }
9053 :
9054 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9055 0 : vec_offset = vec_offsets[j];
9056 :
9057 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9058 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
9059 :
9060 : /* Perform the offset conversion and scaling if necessary. */
9061 0 : if (!strided
9062 0 : && (ls.supported_offset_vectype || ls.supported_scale))
9063 : {
9064 0 : gimple_seq stmts = NULL;
9065 0 : if (ls.supported_offset_vectype)
9066 0 : vec_offset = gimple_convert
9067 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
9068 0 : if (ls.supported_scale)
9069 : {
9070 : /* Only scale the vec_offset if we haven't already. */
9071 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
9072 0 : || j == 0)
9073 : {
9074 0 : tree mult_cst = build_int_cst
9075 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
9076 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
9077 0 : tree mult = build_vector_from_val
9078 0 : (TREE_TYPE (vec_offset), mult_cst);
9079 0 : vec_offset = gimple_build
9080 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
9081 : vec_offset, mult);
9082 : }
9083 0 : scale = size_int (ls.supported_scale);
9084 : }
9085 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9086 : }
9087 :
9088 0 : if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
9089 : {
9090 0 : if (loop_lens)
9091 0 : final_len = vect_get_loop_len (loop_vinfo, gsi,
9092 : loop_lens, num_stmts,
9093 : vectype, j, 1, true);
9094 : else
9095 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9096 :
9097 0 : signed char biasval
9098 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9099 0 : bias = build_int_cst (intQI_type_node, biasval);
9100 0 : if (!final_mask)
9101 : {
9102 0 : mask_vectype = truth_type_for (vectype);
9103 0 : final_mask = build_minus_one_cst (mask_vectype);
9104 : }
9105 : }
9106 :
9107 0 : if (ls.ls_type)
9108 : {
9109 0 : gimple *conv_stmt
9110 0 : = gimple_build_assign (make_ssa_name (vectype),
9111 : VIEW_CONVERT_EXPR,
9112 : build1 (VIEW_CONVERT_EXPR, vectype,
9113 : vec_oprnd));
9114 0 : vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
9115 : gsi);
9116 0 : vec_oprnd = gimple_get_lhs (conv_stmt);
9117 : }
9118 :
9119 0 : gcall *call;
9120 0 : if (final_len && final_mask)
9121 : {
9122 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
9123 0 : call = gimple_build_call_internal (
9124 : IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
9125 : alias_align_ptr,
9126 : vec_offset, scale, vec_oprnd, final_mask, final_len,
9127 : bias);
9128 : else
9129 : /* Non-vector offset indicates that prefer to take
9130 : MASK_LEN_STRIDED_STORE instead of the
9131 : IFN_MASK_SCATTER_STORE with direct stride arg.
9132 : Similar to the gather case we have checked the
9133 : alignment for a scatter already and assume
9134 : that the strided store has the same requirements. */
9135 0 : call = gimple_build_call_internal (
9136 : IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
9137 : vec_offset, vec_oprnd, final_mask, final_len, bias);
9138 : }
9139 0 : else if (final_mask)
9140 0 : call = gimple_build_call_internal
9141 0 : (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
9142 : alias_align_ptr,
9143 : vec_offset, scale, vec_oprnd, final_mask);
9144 : else
9145 0 : call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
9146 : dataref_ptr,
9147 : alias_align_ptr,
9148 : vec_offset,
9149 : scale, vec_oprnd);
9150 0 : gimple_call_set_nothrow (call, true);
9151 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9152 0 : new_stmt = call;
9153 : }
9154 1864 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
9155 : {
9156 : /* The builtin decls path for scatter is legacy, x86 only. */
9157 331 : gcc_assert (nunits.is_constant ()
9158 : && (!final_mask
9159 : || SCALAR_INT_MODE_P
9160 : (TYPE_MODE (TREE_TYPE (final_mask)))));
9161 331 : if (costing_p)
9162 : {
9163 200 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9164 200 : inside_cost
9165 200 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9166 : slp_node, 0, vect_body);
9167 200 : continue;
9168 200 : }
9169 :
9170 131 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9171 131 : poly_uint64 offset_nunits
9172 131 : = TYPE_VECTOR_SUBPARTS (offset_vectype);
9173 131 : if (known_eq (nunits, offset_nunits))
9174 : {
9175 55 : new_stmt = vect_build_one_scatter_store_call
9176 110 : (vinfo, stmt_info, slp_node, gsi,
9177 55 : ls.gs.decl, dataref_ptr, vec_offsets[j],
9178 : vec_oprnd, final_mask);
9179 55 : vect_finish_stmt_generation (vinfo, stmt_info,
9180 : new_stmt, gsi);
9181 : }
9182 76 : else if (known_eq (nunits, offset_nunits * 2))
9183 : {
9184 : /* We have a offset vector with half the number of
9185 : lanes but the builtins will store full vectype
9186 : data from the lower lanes. */
9187 30 : new_stmt = vect_build_one_scatter_store_call
9188 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9189 30 : dataref_ptr, vec_offsets[2 * j],
9190 : vec_oprnd, final_mask);
9191 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9192 : new_stmt, gsi);
9193 30 : int count = nunits.to_constant ();
9194 30 : vec_perm_builder sel (count, count, 1);
9195 30 : sel.quick_grow (count);
9196 412 : for (int i = 0; i < count; ++i)
9197 352 : sel[i] = i | (count / 2);
9198 30 : vec_perm_indices indices (sel, 2, count);
9199 30 : tree perm_mask
9200 30 : = vect_gen_perm_mask_checked (vectype, indices);
9201 30 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
9202 : vec_oprnd, vec_oprnd,
9203 : perm_mask);
9204 30 : vec_oprnd = make_ssa_name (vectype);
9205 30 : gimple_set_lhs (new_stmt, vec_oprnd);
9206 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9207 : new_stmt, gsi);
9208 30 : if (final_mask)
9209 : {
9210 20 : new_stmt = gimple_build_assign (NULL_TREE,
9211 : VEC_UNPACK_HI_EXPR,
9212 : final_mask);
9213 20 : final_mask = make_ssa_name
9214 20 : (truth_type_for (offset_vectype));
9215 20 : gimple_set_lhs (new_stmt, final_mask);
9216 20 : vect_finish_stmt_generation (vinfo, stmt_info,
9217 : new_stmt, gsi);
9218 : }
9219 :
9220 30 : new_stmt = vect_build_one_scatter_store_call
9221 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9222 30 : dataref_ptr, vec_offsets[2 * j + 1],
9223 : vec_oprnd, final_mask);
9224 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9225 : new_stmt, gsi);
9226 30 : }
9227 46 : else if (known_eq (nunits * 2, offset_nunits))
9228 : {
9229 : /* We have a offset vector with double the number of
9230 : lanes. Select the low/high part accordingly. */
9231 46 : vec_offset = vec_offsets[j / 2];
9232 46 : if (j & 1)
9233 : {
9234 23 : int count = offset_nunits.to_constant ();
9235 23 : vec_perm_builder sel (count, count, 1);
9236 23 : sel.quick_grow (count);
9237 286 : for (int i = 0; i < count; ++i)
9238 240 : sel[i] = i | (count / 2);
9239 23 : vec_perm_indices indices (sel, 2, count);
9240 23 : tree perm_mask = vect_gen_perm_mask_checked
9241 23 : (TREE_TYPE (vec_offset), indices);
9242 23 : new_stmt = gimple_build_assign (NULL_TREE,
9243 : VEC_PERM_EXPR,
9244 : vec_offset,
9245 : vec_offset,
9246 : perm_mask);
9247 23 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
9248 23 : gimple_set_lhs (new_stmt, vec_offset);
9249 23 : vect_finish_stmt_generation (vinfo, stmt_info,
9250 : new_stmt, gsi);
9251 23 : }
9252 :
9253 46 : new_stmt = vect_build_one_scatter_store_call
9254 46 : (vinfo, stmt_info, slp_node, gsi,
9255 : ls.gs.decl, dataref_ptr, vec_offset,
9256 : vec_oprnd, final_mask);
9257 46 : vect_finish_stmt_generation (vinfo, stmt_info,
9258 : new_stmt, gsi);
9259 : }
9260 : else
9261 0 : gcc_unreachable ();
9262 : }
9263 : else
9264 : {
9265 : /* Emulated scatter. */
9266 1533 : gcc_assert (!final_mask);
9267 1533 : if (costing_p)
9268 : {
9269 1149 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9270 : /* For emulated scatter N offset vector element extracts
9271 : (we assume the scalar scaling and ptr + offset add is
9272 : consumed by the load). */
9273 1149 : inside_cost
9274 1149 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9275 : slp_node, 0, vect_body);
9276 : /* N scalar stores plus extracting the elements. */
9277 1149 : inside_cost
9278 1149 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9279 : slp_node, 0, vect_body);
9280 1149 : inside_cost
9281 1149 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9282 : slp_node, 0, vect_body);
9283 1149 : continue;
9284 1149 : }
9285 :
9286 384 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9287 384 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
9288 384 : unsigned HOST_WIDE_INT const_offset_nunits
9289 384 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
9290 384 : vec<constructor_elt, va_gc> *ctor_elts;
9291 384 : vec_alloc (ctor_elts, const_nunits);
9292 384 : gimple_seq stmts = NULL;
9293 384 : tree elt_type = TREE_TYPE (vectype);
9294 384 : unsigned HOST_WIDE_INT elt_size
9295 384 : = tree_to_uhwi (TYPE_SIZE (elt_type));
9296 : /* We support offset vectors with more elements
9297 : than the data vector for now. */
9298 384 : unsigned HOST_WIDE_INT factor
9299 : = const_offset_nunits / const_nunits;
9300 384 : vec_offset = vec_offsets[j / factor];
9301 384 : unsigned elt_offset
9302 384 : = (j % factor) * const_nunits;
9303 384 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
9304 384 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9305 384 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
9306 1942 : for (unsigned k = 0; k < const_nunits; ++k)
9307 : {
9308 : /* Compute the offsetted pointer. */
9309 1174 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
9310 : bitsize_int (k + elt_offset));
9311 1174 : tree idx
9312 2348 : = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
9313 1174 : vec_offset, TYPE_SIZE (idx_type), boff);
9314 1174 : idx = gimple_convert (&stmts, sizetype, idx);
9315 1174 : idx = gimple_build (&stmts, MULT_EXPR, sizetype,
9316 : idx, scale);
9317 1174 : tree ptr
9318 1174 : = gimple_build (&stmts, PLUS_EXPR,
9319 1174 : TREE_TYPE (dataref_ptr),
9320 : dataref_ptr, idx);
9321 1174 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
9322 : /* Extract the element to be stored. */
9323 1174 : tree elt
9324 2348 : = gimple_build (&stmts, BIT_FIELD_REF,
9325 1174 : TREE_TYPE (vectype),
9326 1174 : vec_oprnd, TYPE_SIZE (elt_type),
9327 1174 : bitsize_int (k * elt_size));
9328 1174 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9329 1174 : stmts = NULL;
9330 1174 : tree ref
9331 1174 : = build2 (MEM_REF, ltype, ptr,
9332 : build_int_cst (ref_type, 0));
9333 1174 : new_stmt = gimple_build_assign (ref, elt);
9334 1174 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9335 : }
9336 :
9337 384 : slp_node->push_vec_def (new_stmt);
9338 : }
9339 : }
9340 :
9341 1465 : if (costing_p)
9342 : {
9343 985 : if (dump_enabled_p ())
9344 78 : dump_printf_loc (MSG_NOTE, vect_location,
9345 : "vect_model_store_cost: inside_cost = %d, "
9346 : "prologue_cost = %d .\n",
9347 : inside_cost, prologue_cost);
9348 985 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9349 985 : slp_node->data = new vect_load_store_data (std::move (ls));
9350 : }
9351 :
9352 1465 : return true;
9353 1465 : }
9354 :
9355 1352563 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
9356 : || memory_access_type == VMAT_CONTIGUOUS_DOWN
9357 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
9358 :
9359 1352563 : unsigned inside_cost = 0, prologue_cost = 0;
9360 : /* For costing some adjacent vector stores, we'd like to cost with
9361 : the total number of them once instead of cost each one by one. */
9362 1352563 : unsigned int n_adjacent_stores = 0;
9363 1352563 : auto_vec<tree> result_chain (group_size);
9364 1352563 : auto_vec<tree, 1> vec_oprnds;
9365 1352563 : gimple *new_stmt;
9366 1352563 : if (!costing_p)
9367 : {
9368 : /* Get vectorized arguments for SLP_NODE. */
9369 545833 : vect_get_slp_defs (op_node, &vec_oprnds);
9370 545833 : vec_oprnd = vec_oprnds[0];
9371 545833 : if (mask_node)
9372 : {
9373 457 : vect_get_slp_defs (mask_node, &vec_masks);
9374 457 : vec_mask = vec_masks[0];
9375 : }
9376 : }
9377 :
9378 : /* We should have caught mismatched types earlier. */
9379 545833 : gcc_assert (costing_p
9380 : || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
9381 1352563 : bool simd_lane_access_p
9382 1352563 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
9383 1352563 : if (!costing_p
9384 1352563 : && simd_lane_access_p
9385 4364 : && !loop_masks
9386 4364 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
9387 4364 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
9388 4364 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
9389 4364 : && integer_zerop (DR_INIT (first_dr_info->dr))
9390 1356927 : && alias_sets_conflict_p (get_alias_set (aggr_type),
9391 4364 : get_alias_set (TREE_TYPE (ref_type))))
9392 : {
9393 4356 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
9394 4356 : dataref_offset = build_int_cst (ref_type, 0);
9395 : }
9396 1348207 : else if (!costing_p)
9397 1082946 : dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
9398 : simd_lane_access_p ? loop : NULL,
9399 : offset, &dummy, gsi, NULL,
9400 : simd_lane_access_p, dr_increment);
9401 :
9402 1352563 : new_stmt = NULL;
9403 1352563 : gcc_assert (!grouped_store);
9404 3008831 : for (i = 0; i < vec_num; i++)
9405 : {
9406 1656268 : if (!costing_p)
9407 677453 : vec_oprnd = vec_oprnds[i];
9408 :
9409 1656268 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
9410 : {
9411 3371 : if (costing_p)
9412 2227 : inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
9413 : slp_node, 0, vect_body);
9414 : else
9415 : {
9416 1144 : tree perm_mask = perm_mask_for_reverse (vectype);
9417 1144 : tree new_temp = make_ssa_name (vectype);
9418 :
9419 : /* Generate the permute statement. */
9420 1144 : gimple *perm_stmt
9421 1144 : = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
9422 : vec_oprnd, perm_mask);
9423 1144 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9424 :
9425 1144 : perm_stmt = SSA_NAME_DEF_STMT (new_temp);
9426 1656268 : vec_oprnd = new_temp;
9427 : }
9428 : }
9429 :
9430 1656268 : if (costing_p)
9431 : {
9432 978815 : n_adjacent_stores++;
9433 978815 : continue;
9434 : }
9435 :
9436 677453 : tree final_mask = NULL_TREE;
9437 677453 : tree final_len = NULL_TREE;
9438 677453 : tree bias = NULL_TREE;
9439 677453 : if (loop_masks)
9440 78 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
9441 : vec_num, vectype, i);
9442 677453 : if (vec_mask)
9443 658 : vec_mask = vec_masks[i];
9444 658 : if (vec_mask)
9445 658 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
9446 : vec_mask, gsi);
9447 :
9448 677453 : if (i > 0)
9449 : /* Bump the vector pointer. */
9450 131620 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
9451 : dr_bump);
9452 :
9453 677453 : unsigned misalign;
9454 677453 : unsigned HOST_WIDE_INT align;
9455 677453 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
9456 677453 : if (alignment_support_scheme == dr_aligned)
9457 : misalign = 0;
9458 310540 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
9459 : {
9460 162304 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
9461 162304 : misalign = 0;
9462 : }
9463 : else
9464 148236 : misalign = misalignment;
9465 677453 : if (dataref_offset == NULL_TREE
9466 672088 : && TREE_CODE (dataref_ptr) == SSA_NAME)
9467 187729 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, misalign);
9468 677453 : align = least_bit_hwi (misalign | align);
9469 :
9470 : /* Compute IFN when LOOP_LENS or final_mask valid. */
9471 677453 : machine_mode vmode = TYPE_MODE (vectype);
9472 677453 : machine_mode new_vmode = vmode;
9473 677453 : internal_fn partial_ifn = IFN_LAST;
9474 677453 : if (loop_lens)
9475 : {
9476 0 : opt_machine_mode new_ovmode
9477 0 : = get_len_load_store_mode (vmode, false, &partial_ifn);
9478 0 : new_vmode = new_ovmode.require ();
9479 0 : unsigned factor
9480 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
9481 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
9482 : vec_num, vectype, i, factor, true);
9483 : }
9484 677453 : else if (final_mask)
9485 : {
9486 671 : if (!can_vec_mask_load_store_p (vmode,
9487 671 : TYPE_MODE (TREE_TYPE (final_mask)),
9488 : false, &partial_ifn))
9489 0 : gcc_unreachable ();
9490 : }
9491 :
9492 677453 : if (partial_ifn == IFN_MASK_LEN_STORE)
9493 : {
9494 0 : if (!final_len)
9495 : {
9496 : /* Pass VF value to 'len' argument of
9497 : MASK_LEN_STORE if LOOP_LENS is invalid. */
9498 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9499 : }
9500 0 : if (!final_mask)
9501 : {
9502 : /* Pass all ones value to 'mask' argument of
9503 : MASK_LEN_STORE if final_mask is invalid. */
9504 0 : mask_vectype = truth_type_for (vectype);
9505 0 : final_mask = build_minus_one_cst (mask_vectype);
9506 : }
9507 : }
9508 677453 : if (final_len)
9509 : {
9510 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9511 0 : bias = build_int_cst (intQI_type_node, biasval);
9512 : }
9513 :
9514 : /* Arguments are ready. Create the new vector stmt. */
9515 0 : if (final_len)
9516 : {
9517 0 : gcall *call;
9518 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9519 : /* Need conversion if it's wrapped with VnQI. */
9520 0 : if (vmode != new_vmode)
9521 : {
9522 0 : tree new_vtype
9523 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
9524 : new_vmode);
9525 0 : tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
9526 0 : vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
9527 0 : gassign *new_stmt
9528 0 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
9529 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9530 0 : vec_oprnd = var;
9531 : }
9532 :
9533 0 : if (partial_ifn == IFN_MASK_LEN_STORE)
9534 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
9535 : dataref_ptr, ptr, final_mask,
9536 : final_len, bias, vec_oprnd);
9537 : else
9538 0 : call = gimple_build_call_internal (IFN_LEN_STORE, 5,
9539 : dataref_ptr, ptr, final_len,
9540 : bias, vec_oprnd);
9541 0 : gimple_call_set_nothrow (call, true);
9542 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9543 0 : new_stmt = call;
9544 : }
9545 677453 : else if (final_mask)
9546 : {
9547 671 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9548 671 : gcall *call
9549 671 : = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
9550 : ptr, final_mask, vec_oprnd);
9551 671 : gimple_call_set_nothrow (call, true);
9552 671 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9553 671 : new_stmt = call;
9554 : }
9555 : else
9556 : {
9557 676782 : data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
9558 : dataref_offset ? dataref_offset
9559 : : build_int_cst (ref_type, 0));
9560 676782 : if (alignment_support_scheme == dr_aligned
9561 676782 : && align >= TYPE_ALIGN_UNIT (vectype))
9562 : ;
9563 : else
9564 310027 : TREE_TYPE (data_ref)
9565 620054 : = build_aligned_type (TREE_TYPE (data_ref),
9566 : align * BITS_PER_UNIT);
9567 676782 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
9568 676782 : new_stmt = gimple_build_assign (data_ref, vec_oprnd);
9569 676782 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9570 : }
9571 : }
9572 :
9573 1352563 : if (costing_p)
9574 : {
9575 806730 : if (n_adjacent_stores > 0)
9576 806730 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
9577 : alignment_support_scheme, misalignment,
9578 : &inside_cost, cost_vec);
9579 :
9580 : /* When vectorizing a store into the function result assign
9581 : a penalty if the function returns in a multi-register location.
9582 : In this case we assume we'll end up with having to spill the
9583 : vector result and do piecewise loads as a conservative estimate. */
9584 806730 : tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
9585 806730 : if (base
9586 806730 : && (TREE_CODE (base) == RESULT_DECL
9587 756373 : || (DECL_P (base) && cfun_returns (base)))
9588 868998 : && !aggregate_value_p (base, cfun->decl))
9589 : {
9590 11219 : rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
9591 : /* ??? Handle PARALLEL in some way. */
9592 11219 : if (REG_P (reg))
9593 : {
9594 11010 : int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
9595 : /* Assume that a single reg-reg move is possible and cheap,
9596 : do not account for vector to gp register move cost. */
9597 11010 : if (nregs > 1)
9598 : {
9599 : /* Spill. */
9600 10141 : prologue_cost
9601 10141 : += record_stmt_cost (cost_vec, 1, vector_store,
9602 : slp_node, 0, vect_epilogue);
9603 : /* Loads. */
9604 10141 : prologue_cost
9605 10141 : += record_stmt_cost (cost_vec, nregs, scalar_load,
9606 : slp_node, 0, vect_epilogue);
9607 : }
9608 : }
9609 : }
9610 806730 : if (dump_enabled_p ())
9611 14429 : dump_printf_loc (MSG_NOTE, vect_location,
9612 : "vect_model_store_cost: inside_cost = %d, "
9613 : "prologue_cost = %d .\n",
9614 : inside_cost, prologue_cost);
9615 :
9616 806730 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9617 806730 : slp_node->data = new vect_load_store_data (std::move (ls));
9618 : }
9619 :
9620 1352563 : return true;
9621 1385393 : }
9622 :
9623 : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9624 : VECTOR_CST mask. No checks are made that the target platform supports the
9625 : mask, so callers may wish to test can_vec_perm_const_p separately, or use
9626 : vect_gen_perm_mask_checked. */
9627 :
9628 : tree
9629 61869 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
9630 : {
9631 61869 : tree mask_type;
9632 :
9633 61869 : poly_uint64 nunits = sel.length ();
9634 61869 : gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
9635 :
9636 61869 : mask_type = build_vector_type (ssizetype, nunits);
9637 61869 : return vec_perm_indices_to_tree (mask_type, sel);
9638 : }
9639 :
9640 : /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9641 : i.e. that the target supports the pattern _for arbitrary input vectors_. */
9642 :
9643 : tree
9644 59172 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
9645 : {
9646 59172 : machine_mode vmode = TYPE_MODE (vectype);
9647 59172 : gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
9648 59172 : return vect_gen_perm_mask_any (vectype, sel);
9649 : }
9650 :
9651 : /* Given a vector variable X and Y, that was generated for the scalar
9652 : STMT_INFO, generate instructions to permute the vector elements of X and Y
9653 : using permutation mask MASK_VEC, insert them at *GSI and return the
9654 : permuted vector variable. */
9655 :
9656 : static tree
9657 1449 : permute_vec_elements (vec_info *vinfo,
9658 : tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
9659 : gimple_stmt_iterator *gsi)
9660 : {
9661 1449 : tree vectype = TREE_TYPE (x);
9662 1449 : tree perm_dest, data_ref;
9663 1449 : gimple *perm_stmt;
9664 :
9665 1449 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
9666 1449 : if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
9667 1449 : perm_dest = vect_create_destination_var (scalar_dest, vectype);
9668 : else
9669 0 : perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
9670 1449 : data_ref = make_ssa_name (perm_dest);
9671 :
9672 : /* Generate the permute statement. */
9673 1449 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
9674 1449 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9675 :
9676 1449 : return data_ref;
9677 : }
9678 :
9679 : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9680 : inserting them on the loops preheader edge. Returns true if we
9681 : were successful in doing so (and thus STMT_INFO can be moved then),
9682 : otherwise returns false. HOIST_P indicates if we want to hoist the
9683 : definitions of all SSA uses, it would be false when we are costing. */
9684 :
9685 : static bool
9686 3985 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
9687 : {
9688 3985 : ssa_op_iter i;
9689 3985 : use_operand_p use_p;
9690 3985 : auto_vec<use_operand_p, 8> to_hoist;
9691 :
9692 7546 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
9693 : {
9694 3591 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9695 3591 : if (!gimple_nop_p (def_stmt)
9696 3591 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
9697 : {
9698 : /* Make sure we don't need to recurse. While we could do
9699 : so in simple cases when there are more complex use webs
9700 : we don't have an easy way to preserve stmt order to fulfil
9701 : dependencies within them. */
9702 87 : tree op2;
9703 87 : ssa_op_iter i2;
9704 87 : if (gimple_code (def_stmt) == GIMPLE_PHI
9705 87 : || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
9706 : == NULL_DEF_OPERAND_P))
9707 30 : return false;
9708 176 : FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
9709 : {
9710 119 : gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
9711 119 : if (!gimple_nop_p (def_stmt2)
9712 119 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
9713 : return false;
9714 : }
9715 57 : to_hoist.safe_push (use_p);
9716 : }
9717 : }
9718 :
9719 7940 : if (to_hoist.is_empty ())
9720 : return true;
9721 :
9722 33 : if (!hoist_p)
9723 : return true;
9724 :
9725 : /* Instead of moving defs we copy them so we can zero their UID to not
9726 : confuse dominance queries in the preheader. */
9727 3 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
9728 12 : for (use_operand_p use_p : to_hoist)
9729 : {
9730 3 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9731 3 : gimple *copy = gimple_copy (def_stmt);
9732 3 : assign_discriminators_to_stmt
9733 3 : (copy, 0, allocate_copyid_base (gimple_location (copy), 1));
9734 3 : gimple_set_uid (copy, 0);
9735 3 : def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
9736 3 : tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
9737 3 : update_stmt (copy);
9738 3 : def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
9739 3 : SET_DEF (def_p, new_def);
9740 3 : SET_USE (use_p, new_def);
9741 3 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
9742 : }
9743 :
9744 : return true;
9745 3985 : }
9746 :
9747 : /* vectorizable_load.
9748 :
9749 : Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9750 : that can be vectorized.
9751 : If COST_VEC is passed, calculate costs but don't change anything,
9752 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
9753 : it, and insert it at GSI.
9754 : Return true if STMT_INFO is vectorizable in this way. */
9755 :
9756 : static bool
9757 2222435 : vectorizable_load (vec_info *vinfo,
9758 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
9759 : slp_tree slp_node,
9760 : stmt_vector_for_cost *cost_vec)
9761 : {
9762 2222435 : tree scalar_dest;
9763 2222435 : tree vec_dest = NULL;
9764 2222435 : tree data_ref = NULL;
9765 2222435 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
9766 2222435 : class loop *loop = NULL;
9767 2222435 : class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
9768 2222435 : bool nested_in_vect_loop = false;
9769 2222435 : tree elem_type;
9770 : /* Avoid false positive uninitialized warning, see PR110652. */
9771 2222435 : tree new_temp = NULL_TREE;
9772 2222435 : machine_mode mode;
9773 2222435 : tree dummy;
9774 2222435 : tree dataref_ptr = NULL_TREE;
9775 2222435 : tree dataref_offset = NULL_TREE;
9776 2222435 : int i, j;
9777 2222435 : unsigned int group_size;
9778 2222435 : poly_uint64 group_gap_adj;
9779 2222435 : tree msq = NULL_TREE, lsq;
9780 2222435 : tree realignment_token = NULL_TREE;
9781 2222435 : gphi *phi = NULL;
9782 2222435 : bool grouped_load = false;
9783 2222435 : stmt_vec_info first_stmt_info;
9784 2222435 : stmt_vec_info first_stmt_info_for_drptr = NULL;
9785 2222435 : bool compute_in_loop = false;
9786 2222435 : class loop *at_loop;
9787 2222435 : int vec_num;
9788 2222435 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
9789 2222435 : poly_uint64 vf;
9790 2222435 : tree aggr_type;
9791 2222435 : tree ref_type;
9792 2222435 : enum vect_def_type mask_dt = vect_unknown_def_type;
9793 2222435 : enum vect_def_type els_dt = vect_unknown_def_type;
9794 :
9795 2222435 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
9796 : return false;
9797 :
9798 2222435 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
9799 247311 : && cost_vec)
9800 : return false;
9801 :
9802 1975124 : if (!STMT_VINFO_DATA_REF (stmt_info))
9803 : return false;
9804 :
9805 1570084 : tree mask_vectype = NULL_TREE;
9806 1570084 : tree els = NULL_TREE; tree els_vectype = NULL_TREE;
9807 :
9808 1570084 : int mask_index = -1;
9809 1570084 : int els_index = -1;
9810 1570084 : slp_tree mask_node = NULL;
9811 1570084 : slp_tree els_op = NULL;
9812 1570084 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
9813 : {
9814 1565974 : scalar_dest = gimple_assign_lhs (assign);
9815 1565974 : if (TREE_CODE (scalar_dest) != SSA_NAME)
9816 : return false;
9817 :
9818 732158 : tree_code code = gimple_assign_rhs_code (assign);
9819 732158 : if (code != ARRAY_REF
9820 732158 : && code != BIT_FIELD_REF
9821 732158 : && code != INDIRECT_REF
9822 505015 : && code != COMPONENT_REF
9823 505015 : && code != IMAGPART_EXPR
9824 362339 : && code != REALPART_EXPR
9825 362339 : && code != MEM_REF
9826 285 : && TREE_CODE_CLASS (code) != tcc_declaration)
9827 : return false;
9828 : }
9829 : else
9830 : {
9831 4110 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
9832 4110 : if (!call || !gimple_call_internal_p (call))
9833 : return false;
9834 :
9835 4110 : internal_fn ifn = gimple_call_internal_fn (call);
9836 4110 : if (!internal_load_fn_p (ifn))
9837 : return false;
9838 :
9839 2780 : scalar_dest = gimple_call_lhs (call);
9840 2780 : if (!scalar_dest)
9841 : return false;
9842 :
9843 2780 : mask_index = internal_fn_mask_index (ifn);
9844 2780 : if (mask_index >= 0)
9845 2780 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
9846 2780 : if (mask_index >= 0
9847 2780 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
9848 : &mask_node, &mask_dt, &mask_vectype))
9849 : return false;
9850 :
9851 2780 : els_index = internal_fn_else_index (ifn);
9852 2780 : if (els_index >= 0)
9853 2780 : els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
9854 2780 : if (els_index >= 0
9855 2780 : && !vect_is_simple_use (vinfo, slp_node, els_index,
9856 : &els, &els_op, &els_dt, &els_vectype))
9857 : return false;
9858 : }
9859 :
9860 734871 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9861 734871 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9862 :
9863 734871 : if (loop_vinfo)
9864 : {
9865 517227 : loop = LOOP_VINFO_LOOP (loop_vinfo);
9866 517227 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
9867 517227 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9868 : }
9869 : else
9870 : vf = 1;
9871 :
9872 734871 : vec_num = vect_get_num_copies (vinfo, slp_node);
9873 :
9874 : /* FORNOW. This restriction should be relaxed. */
9875 734871 : if (nested_in_vect_loop && vec_num > 1)
9876 : {
9877 319 : if (dump_enabled_p ())
9878 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9879 : "multiple types in nested loop.\n");
9880 : return false;
9881 : }
9882 :
9883 734552 : elem_type = TREE_TYPE (vectype);
9884 734552 : mode = TYPE_MODE (vectype);
9885 :
9886 : /* FORNOW. In some cases can vectorize even if data-type not supported
9887 : (e.g. - data copies). */
9888 734552 : if (!can_implement_p (mov_optab, mode))
9889 : {
9890 0 : if (dump_enabled_p ())
9891 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9892 : "Aligned load, but unsupported type.\n");
9893 : return false;
9894 : }
9895 :
9896 : /* Check if the load is a part of an interleaving chain. */
9897 734552 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
9898 : {
9899 313063 : grouped_load = true;
9900 : /* FORNOW */
9901 313063 : gcc_assert (!nested_in_vect_loop);
9902 313063 : gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
9903 :
9904 313063 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
9905 313063 : group_size = DR_GROUP_SIZE (first_stmt_info);
9906 :
9907 : /* Invalidate assumptions made by dependence analysis when vectorization
9908 : on the unrolled body effectively re-orders stmts. */
9909 313063 : if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
9910 313063 : && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
9911 : STMT_VINFO_MIN_NEG_DIST (stmt_info)))
9912 : {
9913 0 : if (dump_enabled_p ())
9914 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9915 : "cannot perform implicit CSE when performing "
9916 : "group loads with negative dependence distance\n");
9917 : return false;
9918 : }
9919 : }
9920 : else
9921 : group_size = 1;
9922 :
9923 734552 : vect_load_store_data _ls_data{};
9924 734552 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
9925 734552 : if (cost_vec
9926 734552 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
9927 : VLS_LOAD, &ls))
9928 : return false;
9929 : /* Temporary aliases to analysis data, should not be modified through
9930 : these. */
9931 618867 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
9932 618867 : const dr_alignment_support alignment_support_scheme
9933 : = ls.alignment_support_scheme;
9934 618867 : const int misalignment = ls.misalignment;
9935 618867 : const poly_int64 poffset = ls.poffset;
9936 618867 : const vec<int> &elsvals = ls.elsvals;
9937 :
9938 618867 : int maskload_elsval = 0;
9939 618867 : bool need_zeroing = false;
9940 :
9941 : /* We might need to explicitly zero inactive elements if there are
9942 : padding bits in the type that might leak otherwise.
9943 : Refer to PR115336. */
9944 618867 : tree scalar_type = TREE_TYPE (scalar_dest);
9945 618867 : bool type_mode_padding_p
9946 1237734 : = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
9947 :
9948 618867 : if (slp_node->ldst_lanes
9949 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
9950 : {
9951 0 : if (dump_enabled_p ())
9952 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9953 : "discovered load-lane but cannot use it.\n");
9954 : return false;
9955 : }
9956 :
9957 618867 : if (mask_node)
9958 : {
9959 2654 : if (memory_access_type == VMAT_CONTIGUOUS)
9960 : {
9961 2079 : machine_mode vec_mode = TYPE_MODE (vectype);
9962 720 : if (!VECTOR_MODE_P (vec_mode)
9963 4158 : || !can_vec_mask_load_store_p (vec_mode,
9964 2079 : TYPE_MODE (mask_vectype),
9965 : true, NULL, &ls.elsvals))
9966 : return false;
9967 : }
9968 575 : else if (memory_access_type == VMAT_ELEMENTWISE
9969 575 : || memory_access_type == VMAT_STRIDED_SLP)
9970 : {
9971 0 : if (dump_enabled_p ())
9972 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9973 : "unsupported masked strided access.\n");
9974 : return false;
9975 : }
9976 575 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
9977 575 : && !mat_gather_scatter_p (memory_access_type))
9978 : {
9979 32 : if (dump_enabled_p ())
9980 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9981 : "unsupported access type for masked load.\n");
9982 : return false;
9983 : }
9984 543 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
9985 : {
9986 212 : if (dump_enabled_p ())
9987 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9988 : "unsupported masked emulated gather.\n");
9989 : return false;
9990 : }
9991 : }
9992 :
9993 618272 : bool costing_p = cost_vec;
9994 :
9995 618272 : if (costing_p) /* transformation not required. */
9996 : {
9997 461785 : if (loop_vinfo
9998 326730 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
9999 221196 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
10000 : VLS_LOAD, group_size, &ls,
10001 : mask_node, &ls.elsvals);
10002 :
10003 : /* If the type needs padding we must zero inactive elements.
10004 : Check if we can do that with a VEC_COND_EXPR and store the
10005 : elsval we choose in MASKLOAD_ELSVAL. */
10006 461785 : if (ls.elsvals.length ()
10007 61243 : && type_mode_padding_p
10008 7 : && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
10009 61243 : && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
10010 : {
10011 0 : if (dump_enabled_p ())
10012 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10013 : "cannot zero inactive elements.\n");
10014 : return false;
10015 : }
10016 :
10017 461785 : if (mask_node
10018 461785 : && !vect_maybe_update_slp_op_vectype (mask_node,
10019 : mask_vectype))
10020 : {
10021 0 : if (dump_enabled_p ())
10022 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10023 : "incompatible vector types for invariants\n");
10024 : return false;
10025 : }
10026 :
10027 461785 : if (dump_enabled_p ()
10028 26860 : && memory_access_type != VMAT_ELEMENTWISE
10029 26749 : && !mat_gather_scatter_p (memory_access_type)
10030 26434 : && memory_access_type != VMAT_STRIDED_SLP
10031 26434 : && memory_access_type != VMAT_INVARIANT
10032 487252 : && alignment_support_scheme != dr_aligned)
10033 10542 : dump_printf_loc (MSG_NOTE, vect_location,
10034 : "Vectorizing an unaligned access.\n");
10035 :
10036 461785 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10037 0 : vinfo->any_known_not_updated_vssa = true;
10038 : }
10039 :
10040 : /* For now just use the first available else value.
10041 : get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
10042 : select it here if it is supported. */
10043 618272 : if (elsvals.length ())
10044 84526 : maskload_elsval = *elsvals.begin ();
10045 :
10046 618272 : if (dump_enabled_p () && !costing_p)
10047 17014 : dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
10048 :
10049 : /* Transform. */
10050 :
10051 618272 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
10052 618272 : if (!costing_p)
10053 156487 : ensure_base_align (dr_info);
10054 :
10055 618272 : if (memory_access_type == VMAT_INVARIANT)
10056 : {
10057 4207 : gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
10058 : /* If we have versioned for aliasing or the loop doesn't
10059 : have any data dependencies that would preclude this,
10060 : then we are sure this is a loop invariant load and
10061 : thus we can insert it on the preheader edge.
10062 : TODO: hoist_defs_of_uses should ideally be computed
10063 : once at analysis time, remembered and used in the
10064 : transform time. */
10065 8414 : bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
10066 4207 : && !nested_in_vect_loop);
10067 :
10068 4207 : bool uniform_p = true;
10069 17634 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10070 : {
10071 : /* It is unsafe to hoist a conditional load over the conditions that
10072 : make it valid. When early break this means that any invariant load
10073 : can't be hoisted unless it's in the loop header or if we know
10074 : something else has verified the load is valid to do. Alignment
10075 : peeling would do this since getting through the prologue means the
10076 : load was done at least once and so the vector main body is free to
10077 : hoist it. However today GCC will hoist the load above the PFA
10078 : loop. As such that makes it still invalid and so we can't allow it
10079 : today. */
10080 5013 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10081 1210 : && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
10082 6187 : && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
10083 1174 : != loop->header)
10084 : {
10085 1076 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
10086 1076 : && dump_enabled_p ())
10087 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10088 : "not hoisting invariant load due to early break"
10089 : "constraints\n");
10090 1070 : else if (dump_enabled_p ())
10091 26 : dump_printf_loc (MSG_NOTE, vect_location,
10092 : "not hoisting invariant load due to early break"
10093 : "constraints\n");
10094 : hoist_p = false;
10095 : }
10096 :
10097 3969 : hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
10098 5013 : if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
10099 303 : uniform_p = false;
10100 : }
10101 4207 : if (costing_p)
10102 : {
10103 3375 : if (!uniform_p && (!hoist_p || !vf.is_constant ()))
10104 : {
10105 0 : if (dump_enabled_p ())
10106 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10107 : "not vectorizing non-uniform invariant "
10108 : "load\n");
10109 : return false;
10110 : }
10111 1586 : enum vect_cost_model_location cost_loc
10112 3375 : = hoist_p ? vect_prologue : vect_body;
10113 3493 : unsigned int cost = record_stmt_cost (cost_vec,
10114 : uniform_p
10115 118 : ? 1 : SLP_TREE_LANES (slp_node),
10116 : scalar_load,
10117 : slp_node, 0, cost_loc);
10118 3375 : cost += record_stmt_cost (cost_vec, 1,
10119 : uniform_p ? scalar_to_vec : vec_construct,
10120 : slp_node, 0, cost_loc);
10121 3375 : unsigned int prologue_cost = hoist_p ? cost : 0;
10122 1586 : unsigned int inside_cost = hoist_p ? 0 : cost;
10123 3375 : if (dump_enabled_p ())
10124 556 : dump_printf_loc (MSG_NOTE, vect_location,
10125 : "vect_model_load_cost: inside_cost = %d, "
10126 : "prologue_cost = %d .\n",
10127 : inside_cost, prologue_cost);
10128 3375 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10129 3375 : slp_node->data = new vect_load_store_data (std::move (ls));
10130 3375 : return true;
10131 : }
10132 832 : if (hoist_p)
10133 : {
10134 : /* ??? For non-uniform lanes there could be still duplicates.
10135 : We're leaving those to post-vectorizer CSE for the moment. */
10136 630 : auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
10137 2057 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10138 : {
10139 734 : gassign *stmt = as_a <gassign *> (sinfo->stmt);
10140 734 : if (dump_enabled_p ())
10141 379 : dump_printf_loc (MSG_NOTE, vect_location,
10142 : "hoisting out of the vectorized loop: %G",
10143 : (gimple *) stmt);
10144 734 : scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
10145 734 : tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
10146 734 : edge pe = loop_preheader_edge (loop);
10147 734 : gphi *vphi = get_virtual_phi (loop->header);
10148 734 : tree vuse;
10149 734 : if (vphi)
10150 727 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
10151 : else
10152 7 : vuse = gimple_vuse (gsi_stmt (*gsi));
10153 734 : gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
10154 734 : gimple_set_vuse (new_stmt, vuse);
10155 734 : gsi_insert_on_edge_immediate (pe, new_stmt);
10156 734 : hoist_defs_of_uses (new_stmt, loop, true);
10157 734 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
10158 734 : TREE_TYPE (scalar_dest)))
10159 : {
10160 10 : tree tem = make_ssa_name (TREE_TYPE (vectype));
10161 10 : new_stmt = gimple_build_assign (tem,
10162 : NOP_EXPR, scalar_dest);
10163 10 : gsi_insert_on_edge_immediate (pe, new_stmt);
10164 10 : scalar_dest = tem;
10165 : }
10166 734 : scalar_defs.quick_push (scalar_dest);
10167 734 : if (uniform_p)
10168 : break;
10169 : }
10170 630 : if (!uniform_p)
10171 : {
10172 63 : unsigned const_nunits
10173 63 : = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10174 144 : for (j = 0; j < (int) vec_num; ++j)
10175 : {
10176 81 : vec<constructor_elt, va_gc> *v = NULL;
10177 81 : vec_safe_reserve (v, const_nunits, true);
10178 522 : for (unsigned i = 0; i < const_nunits; ++i)
10179 : {
10180 360 : unsigned def_idx
10181 360 : = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
10182 360 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
10183 : scalar_defs[def_idx]);
10184 : }
10185 81 : scalar_dest = build_constructor (vectype, v);
10186 81 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10187 : vectype, NULL);
10188 81 : slp_node->push_vec_def (new_temp);
10189 : }
10190 63 : return true;
10191 : }
10192 567 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10193 : vectype, NULL);
10194 630 : }
10195 : else
10196 : {
10197 202 : gcc_assert (uniform_p);
10198 202 : gimple_stmt_iterator gsi2 = *gsi;
10199 202 : gsi_next (&gsi2);
10200 202 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10201 : vectype, &gsi2);
10202 : }
10203 1613 : for (j = 0; j < (int) vec_num; ++j)
10204 844 : slp_node->push_vec_def (new_temp);
10205 : return true;
10206 : }
10207 :
10208 614065 : if (memory_access_type == VMAT_ELEMENTWISE
10209 614065 : || memory_access_type == VMAT_STRIDED_SLP)
10210 : {
10211 23872 : gimple_stmt_iterator incr_gsi;
10212 23872 : bool insert_after;
10213 23872 : tree offvar = NULL_TREE;
10214 23872 : tree ivstep;
10215 23872 : tree running_off;
10216 23872 : vec<constructor_elt, va_gc> *v = NULL;
10217 23872 : tree stride_base, stride_step = NULL_TREE, alias_off;
10218 : /* Checked by get_load_store_type. */
10219 23872 : unsigned int const_nunits = nunits.to_constant ();
10220 23872 : unsigned HOST_WIDE_INT cst_offset = 0;
10221 23872 : tree dr_offset;
10222 23872 : unsigned int inside_cost = 0;
10223 :
10224 23872 : gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
10225 23872 : gcc_assert (!nested_in_vect_loop);
10226 :
10227 23872 : if (grouped_load)
10228 : {
10229 : /* If we elided a consecutive load permutation, don't
10230 : use the original first statement (which could be elided)
10231 : but the one the load permutation starts with.
10232 : This ensures the stride_base below is correct. */
10233 10883 : if (!ls.subchain_p)
10234 10801 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10235 : else
10236 82 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10237 10883 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10238 10883 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10239 : }
10240 : else
10241 : {
10242 12989 : first_stmt_info = stmt_info;
10243 12989 : first_dr_info = dr_info;
10244 12989 : ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
10245 : }
10246 :
10247 23872 : if (grouped_load)
10248 : {
10249 10883 : if (memory_access_type == VMAT_STRIDED_SLP)
10250 : {
10251 : /* If we elided a consecutive load permutation, adjust
10252 : the group size here. */
10253 4291 : if (!ls.subchain_p)
10254 4209 : group_size = DR_GROUP_SIZE (first_stmt_info);
10255 : else
10256 82 : group_size = SLP_TREE_LANES (slp_node);
10257 : }
10258 : else /* VMAT_ELEMENTWISE */
10259 6592 : group_size = SLP_TREE_LANES (slp_node);
10260 : }
10261 : else
10262 : group_size = 1;
10263 :
10264 23872 : if (!costing_p)
10265 : {
10266 3497 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
10267 3497 : stride_base = fold_build_pointer_plus (
10268 : DR_BASE_ADDRESS (first_dr_info->dr),
10269 : size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
10270 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
10271 3497 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
10272 :
10273 : /* For a load with loop-invariant (but other than power-of-2)
10274 : stride (i.e. not a grouped access) like so:
10275 :
10276 : for (i = 0; i < n; i += stride)
10277 : ... = array[i];
10278 :
10279 : we generate a new induction variable and new accesses to
10280 : form a new vector (or vectors, depending on ncopies):
10281 :
10282 : for (j = 0; ; j += VF*stride)
10283 : tmp1 = array[j];
10284 : tmp2 = array[j + stride];
10285 : ...
10286 : vectemp = {tmp1, tmp2, ...}
10287 : */
10288 :
10289 3497 : tree increment = fold_convert (TREE_TYPE (stride_step),
10290 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
10291 3497 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
10292 : stride_step, increment);
10293 :
10294 3497 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
10295 :
10296 3497 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
10297 3497 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10298 3497 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
10299 : else
10300 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
10301 : true, NULL_TREE, true,
10302 : GSI_SAME_STMT);
10303 3497 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
10304 : insert_after, &offvar, NULL, true);
10305 :
10306 3497 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
10307 : }
10308 :
10309 23872 : running_off = offvar;
10310 23872 : alias_off = build_int_cst (ref_type, 0);
10311 23872 : int nloads = const_nunits;
10312 23872 : int lnel = 1;
10313 23872 : tree ltype = TREE_TYPE (vectype);
10314 23872 : tree lvectype = vectype;
10315 23872 : auto_vec<tree> dr_chain;
10316 : /* ??? Modify local copies of alignment_support_scheme and
10317 : misalignment, but this part of analysis should be done
10318 : earlier and remembered, likewise the chosen load mode. */
10319 23872 : const dr_alignment_support tem = alignment_support_scheme;
10320 23872 : dr_alignment_support alignment_support_scheme = tem;
10321 23872 : const int tem2 = misalignment;
10322 23872 : int misalignment = tem2;
10323 23872 : if (memory_access_type == VMAT_STRIDED_SLP)
10324 : {
10325 17280 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
10326 : /* Use the target vector type if the group size is a multiple
10327 : of it. */
10328 17280 : if (n == const_nunits)
10329 : {
10330 2266 : int mis_align = dr_misalignment (first_dr_info, vectype);
10331 : /* With VF > 1 we advance the DR by step, if that is constant
10332 : and only aligned when performed VF times, DR alignment
10333 : analysis can analyze this as aligned since it assumes
10334 : contiguous accesses. But that is not how we code generate
10335 : here, so adjust for this. */
10336 2266 : if (maybe_gt (vf, 1u)
10337 3634 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10338 3418 : DR_TARGET_ALIGNMENT (first_dr_info)))
10339 216 : mis_align = -1;
10340 2266 : dr_alignment_support dr_align
10341 2266 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
10342 : mis_align);
10343 2266 : if (dr_align == dr_aligned
10344 2266 : || dr_align == dr_unaligned_supported)
10345 : {
10346 17280 : nloads = 1;
10347 17280 : lnel = const_nunits;
10348 17280 : ltype = vectype;
10349 17280 : alignment_support_scheme = dr_align;
10350 17280 : misalignment = mis_align;
10351 : }
10352 : }
10353 : /* Else use the biggest vector we can load the group without
10354 : accessing excess elements. */
10355 15014 : else if (n > 1)
10356 : {
10357 2014 : tree ptype;
10358 2014 : tree vtype
10359 2014 : = vector_vector_composition_type (vectype, const_nunits / n,
10360 : &ptype);
10361 2014 : if (vtype != NULL_TREE)
10362 : {
10363 1976 : dr_alignment_support dr_align;
10364 1976 : int mis_align = 0;
10365 1976 : if (VECTOR_TYPE_P (ptype))
10366 : {
10367 1056 : mis_align = dr_misalignment (first_dr_info, ptype);
10368 1056 : if (maybe_gt (vf, 1u)
10369 2082 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10370 1062 : DR_TARGET_ALIGNMENT (first_dr_info)))
10371 1020 : mis_align = -1;
10372 1056 : dr_align
10373 1056 : = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
10374 : mis_align);
10375 : }
10376 : else
10377 : dr_align = dr_unaligned_supported;
10378 1976 : if (dr_align == dr_aligned
10379 1976 : || dr_align == dr_unaligned_supported)
10380 : {
10381 1976 : nloads = const_nunits / n;
10382 1976 : lnel = n;
10383 1976 : lvectype = vtype;
10384 1976 : ltype = ptype;
10385 1976 : alignment_support_scheme = dr_align;
10386 1976 : misalignment = mis_align;
10387 : }
10388 : }
10389 : }
10390 17280 : unsigned align;
10391 17280 : if (alignment_support_scheme == dr_aligned)
10392 14 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
10393 : else
10394 17266 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
10395 : /* Alignment is at most the access size if we do multiple loads. */
10396 17280 : if (nloads > 1)
10397 15014 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
10398 17280 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
10399 : }
10400 :
10401 23872 : if (costing_p)
10402 : {
10403 : /* Record the composition type for target access during costing. */
10404 20375 : ls.ls_type = lvectype;
10405 20375 : ls.ls_eltype = ltype;
10406 : }
10407 : else
10408 3497 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
10409 :
10410 : /* For SLP permutation support we need to load the whole group,
10411 : not only the number of vector stmts the permutation result
10412 : fits in. */
10413 23872 : int ncopies;
10414 23872 : if (ls.slp_perm)
10415 : {
10416 2871 : gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
10417 : /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10418 : variable VF. */
10419 2871 : unsigned int const_vf = vf.to_constant ();
10420 2871 : ncopies = CEIL (group_size * const_vf, const_nunits);
10421 2871 : dr_chain.create (ncopies);
10422 : }
10423 : else
10424 : ncopies = vec_num;
10425 :
10426 23872 : unsigned int group_el = 0;
10427 23872 : unsigned HOST_WIDE_INT
10428 23872 : elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
10429 23872 : unsigned int n_groups = 0;
10430 : /* For costing some adjacent vector loads, we'd like to cost with
10431 : the total number of them once instead of cost each one by one. */
10432 23872 : unsigned int n_adjacent_loads = 0;
10433 57000 : for (j = 0; j < ncopies; j++)
10434 : {
10435 33128 : if (nloads > 1 && !costing_p)
10436 3209 : vec_alloc (v, nloads);
10437 33128 : gimple *new_stmt = NULL;
10438 139738 : for (i = 0; i < nloads; i++)
10439 : {
10440 106610 : if (costing_p)
10441 : {
10442 : /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10443 : avoid ICE, see PR110776. */
10444 96398 : if (VECTOR_TYPE_P (ltype)
10445 5918 : && memory_access_type != VMAT_ELEMENTWISE)
10446 5918 : n_adjacent_loads++;
10447 : else
10448 90480 : inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
10449 : slp_node, 0, vect_body);
10450 96398 : continue;
10451 : }
10452 10212 : unsigned int load_el = group_el;
10453 : /* For elementwise accesses apply a load permutation directly. */
10454 10212 : if (memory_access_type == VMAT_ELEMENTWISE
10455 10212 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10456 2086 : load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
10457 10212 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
10458 10212 : load_el * elsz + cst_offset);
10459 10212 : tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
10460 10212 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
10461 10212 : new_temp = make_ssa_name (ltype);
10462 10212 : new_stmt = gimple_build_assign (new_temp, data_ref);
10463 10212 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
10464 10212 : if (nloads > 1)
10465 8596 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
10466 :
10467 10212 : group_el += lnel;
10468 10212 : if (group_el == group_size)
10469 : {
10470 9843 : n_groups++;
10471 : /* When doing SLP make sure to not load elements from
10472 : the next vector iteration, those will not be accessed
10473 : so just use the last element again. See PR107451. */
10474 9843 : if (known_lt (n_groups, vf))
10475 : {
10476 6327 : tree newoff = copy_ssa_name (running_off);
10477 6327 : gimple *incr
10478 6327 : = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
10479 : running_off, stride_step);
10480 6327 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
10481 6327 : running_off = newoff;
10482 : }
10483 : group_el = 0;
10484 : }
10485 : }
10486 :
10487 33128 : if (nloads > 1)
10488 : {
10489 24370 : if (costing_p)
10490 21161 : inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
10491 : slp_node, 0, vect_body);
10492 : else
10493 : {
10494 3209 : tree vec_inv = build_constructor (lvectype, v);
10495 3209 : new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
10496 : lvectype, gsi);
10497 3209 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
10498 3209 : if (lvectype != vectype)
10499 : {
10500 398 : new_stmt
10501 398 : = gimple_build_assign (make_ssa_name (vectype),
10502 : VIEW_CONVERT_EXPR,
10503 : build1 (VIEW_CONVERT_EXPR,
10504 : vectype, new_temp));
10505 398 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10506 : gsi);
10507 : }
10508 : }
10509 : }
10510 8758 : else if (!costing_p && ltype != vectype)
10511 : {
10512 1599 : new_stmt = gimple_build_assign (make_ssa_name (vectype),
10513 : VIEW_CONVERT_EXPR,
10514 : build1 (VIEW_CONVERT_EXPR,
10515 : vectype, new_temp));
10516 1599 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10517 : gsi);
10518 : }
10519 :
10520 33128 : if (!costing_p)
10521 : {
10522 4825 : if (ls.slp_perm)
10523 1682 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
10524 : else
10525 3143 : slp_node->push_vec_def (new_stmt);
10526 : }
10527 : }
10528 23872 : if (ls.slp_perm)
10529 : {
10530 2871 : if (costing_p)
10531 : {
10532 2078 : gcc_assert (ls.n_perms != -1U);
10533 2078 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
10534 : slp_node, 0, vect_body);
10535 : }
10536 : else
10537 : {
10538 793 : unsigned n_perms2;
10539 793 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
10540 : false, &n_perms2);
10541 793 : gcc_assert (ls.n_perms == n_perms2);
10542 : }
10543 : }
10544 :
10545 23872 : if (costing_p)
10546 : {
10547 20375 : if (n_adjacent_loads > 0)
10548 2192 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10549 : alignment_support_scheme, misalignment, false,
10550 : &inside_cost, nullptr, cost_vec, cost_vec,
10551 : true);
10552 20375 : if (dump_enabled_p ())
10553 522 : dump_printf_loc (MSG_NOTE, vect_location,
10554 : "vect_model_load_cost: inside_cost = %u, "
10555 : "prologue_cost = 0 .\n",
10556 : inside_cost);
10557 20375 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10558 20375 : slp_node->data = new vect_load_store_data (std::move (ls));
10559 : }
10560 :
10561 23872 : return true;
10562 23872 : }
10563 :
10564 590193 : if (mat_gather_scatter_p (memory_access_type)
10565 590193 : && !ls.ls_type)
10566 : grouped_load = false;
10567 :
10568 587103 : if (grouped_load
10569 590193 : || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10570 : {
10571 264449 : if (grouped_load)
10572 : {
10573 263993 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10574 263993 : group_size = DR_GROUP_SIZE (first_stmt_info);
10575 : }
10576 : else
10577 : {
10578 : first_stmt_info = stmt_info;
10579 : group_size = 1;
10580 : }
10581 : /* For SLP vectorization we directly vectorize a subchain
10582 : without permutation. */
10583 264449 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10584 204614 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10585 : /* For BB vectorization always use the first stmt to base
10586 : the data ref pointer on. */
10587 264449 : if (bb_vinfo)
10588 209462 : first_stmt_info_for_drptr
10589 209462 : = vect_find_first_scalar_stmt_in_slp (slp_node);
10590 :
10591 264449 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10592 264449 : group_gap_adj = 0;
10593 :
10594 : /* VEC_NUM is the number of vect stmts to be created for this group. */
10595 264449 : grouped_load = false;
10596 : /* If an SLP permutation is from N elements to N elements,
10597 : and if one vector holds a whole number of N, we can load
10598 : the inputs to the permutation in the same way as an
10599 : unpermuted sequence. In other cases we need to load the
10600 : whole group, not only the number of vector stmts the
10601 : permutation result fits in. */
10602 264449 : unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
10603 264449 : if (nested_in_vect_loop)
10604 : /* We do not support grouped accesses in a nested loop,
10605 : instead the access is contiguous but it might be
10606 : permuted. No gap adjustment is needed though. */
10607 : ;
10608 264447 : else if (ls.slp_perm
10609 264447 : && (group_size != scalar_lanes
10610 10976 : || !multiple_p (nunits, group_size)))
10611 : {
10612 : /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10613 : variable VF; see vect_transform_slp_perm_load. */
10614 50033 : unsigned int const_vf = vf.to_constant ();
10615 50033 : unsigned int const_nunits = nunits.to_constant ();
10616 50033 : vec_num = CEIL (group_size * const_vf, const_nunits);
10617 50033 : group_gap_adj = vf * group_size - nunits * vec_num;
10618 : }
10619 : else
10620 : {
10621 214414 : group_gap_adj = group_size - scalar_lanes;
10622 : }
10623 :
10624 264449 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10625 : }
10626 : else
10627 : {
10628 325744 : first_stmt_info = stmt_info;
10629 325744 : first_dr_info = dr_info;
10630 325744 : group_size = 1;
10631 325744 : group_gap_adj = 0;
10632 325744 : ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
10633 : }
10634 :
10635 590193 : vec_loop_masks *loop_masks
10636 380731 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10637 590193 : ? &LOOP_VINFO_MASKS (loop_vinfo)
10638 34 : : NULL);
10639 34 : vec_loop_lens *loop_lens
10640 380731 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
10641 : ? &LOOP_VINFO_LENS (loop_vinfo)
10642 0 : : NULL);
10643 :
10644 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10645 : are some difference here. We cannot enable both the lens and masks
10646 : during transform but it is allowed during analysis.
10647 : Shouldn't go with length-based approach if fully masked. */
10648 590193 : if (cost_vec == NULL)
10649 : /* The cost_vec is NULL during transform. */
10650 152158 : gcc_assert ((!loop_lens || !loop_masks));
10651 :
10652 : /* Targets with store-lane instructions must not require explicit
10653 : realignment. vect_supportable_dr_alignment always returns either
10654 : dr_aligned or dr_unaligned_supported for (non-length) masked
10655 : operations. */
10656 590193 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
10657 : && !mask_node
10658 : && !loop_masks)
10659 : || mat_gather_scatter_p (memory_access_type)
10660 : || alignment_support_scheme == dr_aligned
10661 : || alignment_support_scheme == dr_unaligned_supported);
10662 :
10663 : /* In case the vectorization factor (VF) is bigger than the number
10664 : of elements that we can fit in a vectype (nunits), we have to generate
10665 : more than one vector stmt - i.e - we need to "unroll" the
10666 : vector stmt by a factor VF/nunits. In doing so, we record a pointer
10667 : from one copy of the vector stmt to the next, in the field
10668 : STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10669 : stages to find the correct vector defs to be used when vectorizing
10670 : stmts that use the defs of the current stmt. The example below
10671 : illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10672 : need to create 4 vectorized stmts):
10673 :
10674 : before vectorization:
10675 : RELATED_STMT VEC_STMT
10676 : S1: x = memref - -
10677 : S2: z = x + 1 - -
10678 :
10679 : step 1: vectorize stmt S1:
10680 : We first create the vector stmt VS1_0, and, as usual, record a
10681 : pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10682 : Next, we create the vector stmt VS1_1, and record a pointer to
10683 : it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10684 : Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10685 : stmts and pointers:
10686 : RELATED_STMT VEC_STMT
10687 : VS1_0: vx0 = memref0 VS1_1 -
10688 : VS1_1: vx1 = memref1 VS1_2 -
10689 : VS1_2: vx2 = memref2 VS1_3 -
10690 : VS1_3: vx3 = memref3 - -
10691 : S1: x = load - VS1_0
10692 : S2: z = x + 1 - -
10693 : */
10694 :
10695 : /* If the data reference is aligned (dr_aligned) or potentially unaligned
10696 : on a target that supports unaligned accesses (dr_unaligned_supported)
10697 : we generate the following code:
10698 : p = initial_addr;
10699 : indx = 0;
10700 : loop {
10701 : p = p + indx * vectype_size;
10702 : vec_dest = *(p);
10703 : indx = indx + 1;
10704 : }
10705 :
10706 : Otherwise, the data reference is potentially unaligned on a target that
10707 : does not support unaligned accesses (dr_explicit_realign_optimized) -
10708 : then generate the following code, in which the data in each iteration is
10709 : obtained by two vector loads, one from the previous iteration, and one
10710 : from the current iteration:
10711 : p1 = initial_addr;
10712 : msq_init = *(floor(p1))
10713 : p2 = initial_addr + VS - 1;
10714 : realignment_token = call target_builtin;
10715 : indx = 0;
10716 : loop {
10717 : p2 = p2 + indx * vectype_size
10718 : lsq = *(floor(p2))
10719 : vec_dest = realign_load (msq, lsq, realignment_token)
10720 : indx = indx + 1;
10721 : msq = lsq;
10722 : } */
10723 :
10724 : /* If the misalignment remains the same throughout the execution of the
10725 : loop, we can create the init_addr and permutation mask at the loop
10726 : preheader. Otherwise, it needs to be created inside the loop.
10727 : This can only occur when vectorizing memory accesses in the inner-loop
10728 : nested within an outer-loop that is being vectorized. */
10729 :
10730 590193 : if (nested_in_vect_loop
10731 590193 : && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
10732 1234 : GET_MODE_SIZE (TYPE_MODE (vectype))))
10733 : {
10734 195 : gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
10735 : compute_in_loop = true;
10736 : }
10737 :
10738 590193 : bool diff_first_stmt_info
10739 590193 : = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
10740 :
10741 590193 : tree offset = NULL_TREE;
10742 590193 : if ((alignment_support_scheme == dr_explicit_realign_optimized
10743 590193 : || alignment_support_scheme == dr_explicit_realign)
10744 0 : && !compute_in_loop)
10745 : {
10746 : /* If we have different first_stmt_info, we can't set up realignment
10747 : here, since we can't guarantee first_stmt_info DR has been
10748 : initialized yet, use first_stmt_info_for_drptr DR by bumping the
10749 : distance from first_stmt_info DR instead as below. */
10750 0 : if (!costing_p)
10751 : {
10752 0 : if (!diff_first_stmt_info)
10753 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
10754 : &realignment_token,
10755 : alignment_support_scheme, NULL_TREE,
10756 : &at_loop);
10757 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
10758 : {
10759 0 : phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
10760 0 : offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
10761 : size_one_node);
10762 0 : gcc_assert (!first_stmt_info_for_drptr);
10763 : }
10764 : }
10765 : }
10766 : else
10767 590193 : at_loop = loop;
10768 :
10769 590193 : if (!known_eq (poffset, 0))
10770 4674 : offset = (offset
10771 4674 : ? size_binop (PLUS_EXPR, offset, size_int (poffset))
10772 4674 : : size_int (poffset));
10773 :
10774 590193 : tree dr_increment;
10775 590193 : tree dr_bump;
10776 590193 : tree vec_offset = NULL_TREE;
10777 :
10778 590193 : auto_vec<tree> vec_offsets;
10779 590193 : auto_vec<tree> vec_masks;
10780 590193 : if (mask_node && !costing_p)
10781 628 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
10782 : &vec_masks);
10783 :
10784 590193 : tree vec_mask = NULL_TREE;
10785 590193 : tree vec_els = NULL_TREE;
10786 590193 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10787 : {
10788 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
10789 :
10790 0 : gcc_assert (alignment_support_scheme == dr_aligned
10791 : || alignment_support_scheme == dr_unaligned_supported);
10792 :
10793 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
10794 0 : if (!costing_p)
10795 : {
10796 0 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
10797 : memory_access_type);
10798 0 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
10799 : memory_access_type);
10800 : }
10801 :
10802 0 : unsigned int inside_cost = 0, prologue_cost = 0;
10803 : /* For costing some adjacent vector loads, we'd like to cost with
10804 : the total number of them once instead of cost each one by one. */
10805 0 : unsigned int n_adjacent_loads = 0;
10806 0 : int ncopies = vec_num / group_size;
10807 0 : for (j = 0; j < ncopies; j++)
10808 : {
10809 0 : if (costing_p)
10810 : {
10811 : /* An IFN_LOAD_LANES will load all its vector results,
10812 : regardless of which ones we actually need. Account
10813 : for the cost of unused results. */
10814 0 : if (first_stmt_info == stmt_info)
10815 : {
10816 0 : unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
10817 0 : stmt_vec_info next_stmt_info = first_stmt_info;
10818 0 : do
10819 : {
10820 0 : gaps -= 1;
10821 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
10822 : }
10823 0 : while (next_stmt_info);
10824 0 : if (gaps)
10825 : {
10826 0 : if (dump_enabled_p ())
10827 0 : dump_printf_loc (MSG_NOTE, vect_location,
10828 : "vect_model_load_cost: %d "
10829 : "unused vectors.\n",
10830 : gaps);
10831 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
10832 : alignment_support_scheme,
10833 : misalignment, false, &inside_cost,
10834 : &prologue_cost, cost_vec, cost_vec,
10835 : true);
10836 : }
10837 : }
10838 0 : n_adjacent_loads++;
10839 0 : continue;
10840 0 : }
10841 :
10842 : /* 1. Create the vector or array pointer update chain. */
10843 0 : if (j == 0)
10844 0 : dataref_ptr
10845 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10846 : at_loop, offset, &dummy, gsi,
10847 : NULL, false, dr_increment);
10848 : else
10849 : {
10850 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
10851 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10852 : stmt_info, dr_bump);
10853 : }
10854 0 : if (mask_node)
10855 0 : vec_mask = vec_masks[j];
10856 :
10857 0 : tree vec_array = create_vector_array (vectype, group_size);
10858 :
10859 0 : tree final_mask = NULL_TREE;
10860 0 : tree final_len = NULL_TREE;
10861 0 : tree bias = NULL_TREE;
10862 0 : if (loop_masks)
10863 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10864 : ncopies, vectype, j);
10865 0 : if (vec_mask)
10866 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
10867 : vec_mask, gsi);
10868 :
10869 0 : if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
10870 : {
10871 0 : if (loop_lens)
10872 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
10873 : ncopies, vectype, j, 1, true);
10874 : else
10875 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
10876 0 : signed char biasval
10877 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
10878 0 : bias = build_int_cst (intQI_type_node, biasval);
10879 0 : if (!final_mask)
10880 : {
10881 0 : mask_vectype = truth_type_for (vectype);
10882 0 : final_mask = build_minus_one_cst (mask_vectype);
10883 : }
10884 : }
10885 :
10886 0 : if (final_mask)
10887 : {
10888 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
10889 0 : if (type_mode_padding_p
10890 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
10891 0 : need_zeroing = true;
10892 : }
10893 :
10894 0 : gcall *call;
10895 0 : if (final_len && final_mask)
10896 : {
10897 : /* Emit:
10898 : VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10899 : VEC_MASK, LEN, BIAS). */
10900 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10901 0 : tree alias_ptr = build_int_cst (ref_type, align);
10902 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
10903 : dataref_ptr, alias_ptr,
10904 : final_mask, vec_els,
10905 : final_len, bias);
10906 : }
10907 0 : else if (final_mask)
10908 : {
10909 : /* Emit:
10910 : VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10911 : VEC_MASK). */
10912 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10913 0 : tree alias_ptr = build_int_cst (ref_type, align);
10914 0 : call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
10915 : dataref_ptr, alias_ptr,
10916 : final_mask, vec_els);
10917 : }
10918 : else
10919 : {
10920 : /* Emit:
10921 : VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10922 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
10923 0 : call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
10924 : }
10925 0 : gimple_call_set_lhs (call, vec_array);
10926 0 : gimple_call_set_nothrow (call, true);
10927 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
10928 :
10929 : /* Extract each vector into an SSA_NAME. */
10930 0 : for (unsigned i = 0; i < group_size; i++)
10931 : {
10932 0 : new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
10933 : vec_array, i, need_zeroing,
10934 : final_mask);
10935 0 : slp_node->push_vec_def (new_temp);
10936 : }
10937 :
10938 : /* Record that VEC_ARRAY is now dead. */
10939 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
10940 : }
10941 :
10942 0 : if (costing_p)
10943 : {
10944 0 : if (n_adjacent_loads > 0)
10945 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10946 : alignment_support_scheme, misalignment, false,
10947 : &inside_cost, &prologue_cost, cost_vec,
10948 : cost_vec, true);
10949 0 : if (dump_enabled_p ())
10950 0 : dump_printf_loc (MSG_NOTE, vect_location,
10951 : "vect_model_load_cost: inside_cost = %u, "
10952 : "prologue_cost = %u .\n",
10953 : inside_cost, prologue_cost);
10954 0 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10955 0 : slp_node->data = new vect_load_store_data (std::move (ls));
10956 : }
10957 :
10958 0 : return true;
10959 : }
10960 :
10961 590193 : if (mat_gather_scatter_p (memory_access_type))
10962 : {
10963 3090 : gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
10964 :
10965 3090 : auto_vec<tree> dr_chain (vec_num);
10966 :
10967 : /* If we pun the original vectype the loads as well as costing, length,
10968 : etc. is performed with the new type. After loading we VIEW_CONVERT
10969 : the data to the original vectype. */
10970 3090 : tree original_vectype = vectype;
10971 3090 : if (ls.ls_type)
10972 0 : vectype = ls.ls_type;
10973 :
10974 : /* 1. Create the vector or array pointer update chain. */
10975 3090 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10976 : {
10977 3090 : aggr_type = NULL_TREE;
10978 3090 : dr_increment = NULL_TREE;
10979 3090 : if (!costing_p)
10980 789 : vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
10981 : &vec_offsets);
10982 : }
10983 : else
10984 : {
10985 0 : aggr_type = elem_type;
10986 0 : if (!costing_p)
10987 : {
10988 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
10989 : ls.strided_offset_vectype,
10990 : loop_vinfo, gsi,
10991 : &dr_increment, &dr_bump,
10992 : &vec_offset);
10993 0 : dataref_ptr
10994 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10995 : at_loop, offset, &dummy, gsi,
10996 : NULL, false, dr_increment);
10997 : }
10998 : }
10999 :
11000 3090 : unsigned int inside_cost = 0, prologue_cost = 0;
11001 :
11002 3090 : gimple *new_stmt = NULL;
11003 6969 : for (i = 0; i < vec_num; i++)
11004 : {
11005 3879 : tree final_mask = NULL_TREE;
11006 3879 : tree final_len = NULL_TREE;
11007 3879 : tree bias = NULL_TREE;
11008 3879 : if (!costing_p)
11009 : {
11010 1012 : if (mask_node)
11011 156 : vec_mask = vec_masks[i];
11012 1012 : if (loop_masks)
11013 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11014 : vec_num, vectype, i);
11015 1012 : if (vec_mask)
11016 156 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11017 : final_mask, vec_mask, gsi);
11018 :
11019 1012 : if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11020 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11021 : stmt_info, dr_bump);
11022 : }
11023 :
11024 : /* 2. Create the vector-load in the loop. */
11025 3879 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
11026 3879 : tree alias_align_ptr = build_int_cst (ref_type, align);
11027 3879 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
11028 : {
11029 0 : if (costing_p)
11030 : {
11031 0 : if (ls.supported_offset_vectype
11032 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
11033 : vec_offset))
11034 0 : inside_cost
11035 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11036 : slp_node, 0, vect_body);
11037 0 : if (ls.supported_scale)
11038 0 : inside_cost
11039 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11040 : slp_node, 0, vect_body);
11041 :
11042 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11043 0 : inside_cost
11044 0 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11045 : slp_node, 0, vect_body);
11046 0 : continue;
11047 0 : }
11048 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11049 0 : vec_offset = vec_offsets[i];
11050 0 : tree zero = build_zero_cst (vectype);
11051 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11052 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
11053 :
11054 : /* Perform the offset conversion and scaling if necessary. */
11055 0 : if (!strided
11056 0 : && (ls.supported_offset_vectype || ls.supported_scale))
11057 : {
11058 0 : gimple_seq stmts = NULL;
11059 0 : if (ls.supported_offset_vectype)
11060 0 : vec_offset = gimple_convert
11061 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
11062 0 : if (ls.supported_scale)
11063 : {
11064 : /* Only scale the vec_offset if we haven't already. */
11065 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
11066 0 : || i == 0)
11067 : {
11068 0 : tree mult_cst = build_int_cst
11069 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
11070 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
11071 0 : tree mult = build_vector_from_val
11072 0 : (TREE_TYPE (vec_offset), mult_cst);
11073 0 : vec_offset = gimple_build
11074 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
11075 : vec_offset, mult);
11076 : }
11077 0 : scale = size_int (ls.supported_scale);
11078 : }
11079 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11080 : }
11081 :
11082 0 : if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
11083 : {
11084 0 : if (loop_lens)
11085 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11086 : vec_num, vectype, i, 1, true);
11087 : else
11088 0 : final_len = build_int_cst (sizetype,
11089 0 : TYPE_VECTOR_SUBPARTS (vectype));
11090 0 : signed char biasval
11091 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11092 0 : bias = build_int_cst (intQI_type_node, biasval);
11093 0 : if (!final_mask)
11094 : {
11095 0 : mask_vectype = truth_type_for (vectype);
11096 0 : final_mask = build_minus_one_cst (mask_vectype);
11097 : }
11098 : }
11099 :
11100 0 : if (final_mask)
11101 : {
11102 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11103 0 : if (type_mode_padding_p
11104 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11105 0 : need_zeroing = true;
11106 : }
11107 :
11108 0 : gcall *call;
11109 0 : if (final_len && final_mask)
11110 : {
11111 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
11112 0 : call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
11113 : 9, dataref_ptr,
11114 : alias_align_ptr,
11115 : vec_offset, scale, zero,
11116 : final_mask, vec_els,
11117 : final_len, bias);
11118 : else
11119 : /* Non-vector offset indicates that prefer to take
11120 : MASK_LEN_STRIDED_LOAD instead of the
11121 : MASK_LEN_GATHER_LOAD with direct stride arg. */
11122 0 : call = gimple_build_call_internal
11123 0 : (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
11124 : vec_offset, zero, final_mask, vec_els, final_len,
11125 : bias);
11126 : }
11127 0 : else if (final_mask)
11128 0 : call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
11129 : 7, dataref_ptr,
11130 : alias_align_ptr,
11131 : vec_offset, scale,
11132 : zero, final_mask, vec_els);
11133 : else
11134 0 : call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
11135 : dataref_ptr,
11136 : alias_align_ptr,
11137 : vec_offset, scale, zero);
11138 0 : gimple_call_set_nothrow (call, true);
11139 0 : new_stmt = call;
11140 0 : data_ref = NULL_TREE;
11141 : }
11142 3879 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
11143 : {
11144 : /* The builtin decls path for gather is legacy, x86 only. */
11145 848 : gcc_assert (!final_len && nunits.is_constant ());
11146 848 : if (costing_p)
11147 : {
11148 566 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11149 566 : inside_cost
11150 566 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11151 : slp_node, 0, vect_body);
11152 566 : continue;
11153 566 : }
11154 282 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11155 282 : poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
11156 282 : if (known_eq (nunits, offset_nunits))
11157 : {
11158 133 : new_stmt = vect_build_one_gather_load_call
11159 133 : (vinfo, stmt_info, slp_node, vectype, gsi,
11160 133 : ls.gs.decl, dataref_ptr, vec_offsets[i],
11161 : final_mask);
11162 133 : data_ref = NULL_TREE;
11163 : }
11164 149 : else if (known_eq (nunits, offset_nunits * 2))
11165 : {
11166 : /* We have a offset vector with half the number of
11167 : lanes but the builtins will produce full vectype
11168 : data with just the lower lanes filled. */
11169 63 : new_stmt = vect_build_one_gather_load_call
11170 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11171 63 : ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
11172 : final_mask);
11173 63 : tree low = make_ssa_name (vectype);
11174 63 : gimple_set_lhs (new_stmt, low);
11175 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11176 :
11177 : /* now put upper half of final_mask in final_mask low. */
11178 63 : if (final_mask
11179 63 : && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
11180 : {
11181 11 : int count = nunits.to_constant ();
11182 11 : vec_perm_builder sel (count, count, 1);
11183 11 : sel.quick_grow (count);
11184 98 : for (int i = 0; i < count; ++i)
11185 76 : sel[i] = i | (count / 2);
11186 11 : vec_perm_indices indices (sel, 2, count);
11187 11 : tree perm_mask = vect_gen_perm_mask_checked
11188 11 : (TREE_TYPE (final_mask), indices);
11189 11 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11190 : final_mask, final_mask,
11191 : perm_mask);
11192 11 : final_mask = make_ssa_name (TREE_TYPE (final_mask));
11193 11 : gimple_set_lhs (new_stmt, final_mask);
11194 11 : vect_finish_stmt_generation (vinfo, stmt_info,
11195 : new_stmt, gsi);
11196 11 : }
11197 52 : else if (final_mask)
11198 : {
11199 24 : new_stmt = gimple_build_assign (NULL_TREE,
11200 : VEC_UNPACK_HI_EXPR,
11201 : final_mask);
11202 24 : final_mask = make_ssa_name
11203 24 : (truth_type_for (offset_vectype));
11204 24 : gimple_set_lhs (new_stmt, final_mask);
11205 24 : vect_finish_stmt_generation (vinfo, stmt_info,
11206 : new_stmt, gsi);
11207 : }
11208 :
11209 63 : new_stmt = vect_build_one_gather_load_call
11210 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11211 : ls.gs.decl, dataref_ptr,
11212 63 : vec_offsets[2 * i + 1], final_mask);
11213 63 : tree high = make_ssa_name (vectype);
11214 63 : gimple_set_lhs (new_stmt, high);
11215 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11216 :
11217 : /* compose low + high. */
11218 63 : int count = nunits.to_constant ();
11219 63 : vec_perm_builder sel (count, count, 1);
11220 63 : sel.quick_grow (count);
11221 710 : for (int i = 0; i < count; ++i)
11222 584 : sel[i] = i < count / 2 ? i : i + count / 2;
11223 63 : vec_perm_indices indices (sel, 2, count);
11224 63 : tree perm_mask
11225 63 : = vect_gen_perm_mask_checked (vectype, indices);
11226 63 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11227 : low, high, perm_mask);
11228 63 : data_ref = NULL_TREE;
11229 63 : }
11230 86 : else if (known_eq (nunits * 2, offset_nunits))
11231 : {
11232 : /* We have a offset vector with double the number of
11233 : lanes. Select the low/high part accordingly. */
11234 86 : vec_offset = vec_offsets[i / 2];
11235 86 : if (i & 1)
11236 : {
11237 43 : int count = offset_nunits.to_constant ();
11238 43 : vec_perm_builder sel (count, count, 1);
11239 43 : sel.quick_grow (count);
11240 506 : for (int i = 0; i < count; ++i)
11241 420 : sel[i] = i | (count / 2);
11242 43 : vec_perm_indices indices (sel, 2, count);
11243 43 : tree perm_mask = vect_gen_perm_mask_checked
11244 43 : (TREE_TYPE (vec_offset), indices);
11245 43 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11246 : vec_offset, vec_offset,
11247 : perm_mask);
11248 43 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
11249 43 : gimple_set_lhs (new_stmt, vec_offset);
11250 43 : vect_finish_stmt_generation (vinfo, stmt_info,
11251 : new_stmt, gsi);
11252 43 : }
11253 86 : new_stmt = vect_build_one_gather_load_call
11254 86 : (vinfo, stmt_info, slp_node, vectype, gsi,
11255 : ls.gs.decl,
11256 : dataref_ptr, vec_offset, final_mask);
11257 86 : data_ref = NULL_TREE;
11258 : }
11259 : else
11260 0 : gcc_unreachable ();
11261 : }
11262 : else
11263 : {
11264 : /* Emulated gather-scatter. */
11265 3031 : gcc_assert (!final_mask);
11266 3031 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
11267 3031 : if (costing_p)
11268 : {
11269 : /* For emulated gathers N offset vector element
11270 : offset add is consumed by the load). */
11271 2301 : inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
11272 : slp_node, 0, vect_body);
11273 : /* N scalar loads plus gathering them into a
11274 : vector. */
11275 2301 : inside_cost
11276 2301 : = record_stmt_cost (cost_vec, const_nunits, scalar_load,
11277 : slp_node, 0, vect_body);
11278 2301 : inside_cost
11279 2301 : = record_stmt_cost (cost_vec, 1, vec_construct,
11280 : slp_node, 0, vect_body);
11281 2301 : continue;
11282 : }
11283 730 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11284 730 : unsigned HOST_WIDE_INT const_offset_nunits
11285 730 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
11286 730 : vec<constructor_elt, va_gc> *ctor_elts;
11287 730 : vec_alloc (ctor_elts, const_nunits);
11288 730 : gimple_seq stmts = NULL;
11289 : /* We support offset vectors with more elements
11290 : than the data vector for now. */
11291 730 : unsigned HOST_WIDE_INT factor
11292 : = const_offset_nunits / const_nunits;
11293 730 : vec_offset = vec_offsets[i / factor];
11294 730 : unsigned elt_offset = (i % factor) * const_nunits;
11295 730 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
11296 730 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11297 730 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
11298 3656 : for (unsigned k = 0; k < const_nunits; ++k)
11299 : {
11300 2196 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
11301 : bitsize_int (k + elt_offset));
11302 6588 : tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
11303 2196 : vec_offset, TYPE_SIZE (idx_type),
11304 : boff);
11305 2196 : idx = gimple_convert (&stmts, sizetype, idx);
11306 2196 : idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
11307 2196 : tree ptr = gimple_build (&stmts, PLUS_EXPR,
11308 2196 : TREE_TYPE (dataref_ptr),
11309 : dataref_ptr, idx);
11310 2196 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
11311 2196 : tree elt = make_ssa_name (TREE_TYPE (vectype));
11312 2196 : tree ref = build2 (MEM_REF, ltype, ptr,
11313 : build_int_cst (ref_type, 0));
11314 2196 : new_stmt = gimple_build_assign (elt, ref);
11315 4392 : gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
11316 2196 : gimple_seq_add_stmt (&stmts, new_stmt);
11317 2196 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
11318 : }
11319 730 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11320 730 : new_stmt = gimple_build_assign (NULL_TREE,
11321 : build_constructor (vectype,
11322 : ctor_elts));
11323 730 : data_ref = NULL_TREE;
11324 : }
11325 :
11326 1012 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11327 : /* DATA_REF is null if we've already built the statement. */
11328 1012 : if (data_ref)
11329 : {
11330 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11331 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11332 : }
11333 2024 : new_temp = (need_zeroing
11334 1012 : ? make_ssa_name (vectype)
11335 1012 : : make_ssa_name (vec_dest, new_stmt));
11336 1012 : gimple_set_lhs (new_stmt, new_temp);
11337 1012 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11338 :
11339 : /* If we need to explicitly zero inactive elements emit a
11340 : VEC_COND_EXPR that does so. */
11341 1012 : if (need_zeroing)
11342 : {
11343 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11344 : vectype);
11345 :
11346 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11347 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11348 : final_mask, new_temp, vec_els);
11349 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11350 0 : new_temp = new_temp2;
11351 : }
11352 :
11353 1012 : if (ls.ls_type)
11354 : {
11355 0 : new_stmt = gimple_build_assign (make_ssa_name
11356 : (original_vectype),
11357 : VIEW_CONVERT_EXPR,
11358 : build1 (VIEW_CONVERT_EXPR,
11359 : original_vectype,
11360 : new_temp));
11361 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11362 : }
11363 :
11364 : /* Store vector loads in the corresponding SLP_NODE. */
11365 1012 : if (!costing_p)
11366 : {
11367 1012 : if (ls.slp_perm)
11368 0 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
11369 : else
11370 1012 : slp_node->push_vec_def (new_stmt);
11371 : }
11372 : }
11373 :
11374 3090 : if (ls.slp_perm)
11375 : {
11376 0 : if (costing_p)
11377 : {
11378 0 : gcc_assert (ls.n_perms != -1U);
11379 0 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11380 : slp_node, 0, vect_body);
11381 : }
11382 : else
11383 : {
11384 0 : unsigned n_perms2;
11385 0 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
11386 : false, &n_perms2);
11387 0 : gcc_assert (ls.n_perms == n_perms2);
11388 : }
11389 : }
11390 :
11391 3090 : if (costing_p)
11392 : {
11393 2301 : if (dump_enabled_p ())
11394 315 : dump_printf_loc (MSG_NOTE, vect_location,
11395 : "vect_model_load_cost: inside_cost = %u, "
11396 : "prologue_cost = %u .\n",
11397 : inside_cost, prologue_cost);
11398 2301 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11399 2301 : slp_node->data = new vect_load_store_data (std::move (ls));
11400 : }
11401 3090 : return true;
11402 3090 : }
11403 :
11404 587103 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
11405 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
11406 :
11407 587103 : aggr_type = vectype;
11408 587103 : if (!costing_p)
11409 : {
11410 151369 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
11411 : memory_access_type);
11412 151369 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
11413 : memory_access_type);
11414 : }
11415 :
11416 587103 : poly_uint64 group_elt = 0;
11417 587103 : unsigned int inside_cost = 0, prologue_cost = 0;
11418 : /* For costing some adjacent vector loads, we'd like to cost with
11419 : the total number of them once instead of cost each one by one. */
11420 587103 : unsigned int n_adjacent_loads = 0;
11421 :
11422 : /* 1. Create the vector or array pointer update chain. */
11423 587103 : if (!costing_p)
11424 : {
11425 151369 : bool simd_lane_access_p
11426 151369 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
11427 151369 : if (simd_lane_access_p
11428 1633 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
11429 1633 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
11430 1633 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
11431 1633 : && integer_zerop (DR_INIT (first_dr_info->dr))
11432 1633 : && alias_sets_conflict_p (get_alias_set (aggr_type),
11433 1633 : get_alias_set (TREE_TYPE (ref_type)))
11434 151369 : && (alignment_support_scheme == dr_aligned
11435 1633 : || alignment_support_scheme == dr_unaligned_supported))
11436 : {
11437 1633 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
11438 1633 : dataref_offset = build_int_cst (ref_type, 0);
11439 : }
11440 149736 : else if (diff_first_stmt_info)
11441 : {
11442 3382 : dataref_ptr
11443 3382 : = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
11444 : aggr_type, at_loop, offset, &dummy,
11445 : gsi, NULL, simd_lane_access_p,
11446 : dr_increment);
11447 : /* Adjust the pointer by the difference to first_stmt. */
11448 3382 : data_reference_p ptrdr
11449 : = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
11450 3382 : tree diff = fold_convert (sizetype,
11451 : size_binop (MINUS_EXPR,
11452 : DR_INIT (first_dr_info->dr),
11453 : DR_INIT (ptrdr)));
11454 3382 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11455 : stmt_info, diff);
11456 3382 : if (alignment_support_scheme == dr_explicit_realign)
11457 : {
11458 0 : msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
11459 : vectype, gsi,
11460 : &realignment_token,
11461 : alignment_support_scheme,
11462 : dataref_ptr, &at_loop);
11463 0 : gcc_assert (!compute_in_loop);
11464 : }
11465 : }
11466 : else
11467 146354 : dataref_ptr
11468 146354 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11469 : at_loop,
11470 : offset, &dummy, gsi, NULL,
11471 : simd_lane_access_p, dr_increment);
11472 : }
11473 :
11474 587103 : auto_vec<tree> dr_chain;
11475 587103 : if (grouped_load || ls.slp_perm)
11476 59835 : dr_chain.create (vec_num);
11477 :
11478 587103 : gimple *new_stmt = NULL;
11479 1538808 : for (i = 0; i < vec_num; i++)
11480 : {
11481 951705 : tree final_mask = NULL_TREE;
11482 951705 : tree final_len = NULL_TREE;
11483 951705 : tree bias = NULL_TREE;
11484 :
11485 951705 : if (!costing_p)
11486 : {
11487 240086 : if (mask_node)
11488 651 : vec_mask = vec_masks[i];
11489 240086 : if (loop_masks)
11490 51 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11491 : vec_num, vectype, i);
11492 240086 : if (vec_mask)
11493 651 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11494 : final_mask, vec_mask, gsi);
11495 :
11496 240086 : if (i > 0)
11497 88717 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11498 : dr_bump);
11499 : }
11500 :
11501 : /* 2. Create the vector-load in the loop. */
11502 951705 : switch (alignment_support_scheme)
11503 : {
11504 951705 : case dr_aligned:
11505 951705 : case dr_unaligned_supported:
11506 951705 : {
11507 951705 : if (costing_p)
11508 : break;
11509 :
11510 240086 : unsigned int misalign;
11511 240086 : unsigned HOST_WIDE_INT align;
11512 240086 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
11513 240086 : if (alignment_support_scheme == dr_aligned)
11514 : misalign = 0;
11515 155882 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
11516 : {
11517 118578 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
11518 118578 : misalign = 0;
11519 : }
11520 : else
11521 37304 : misalign = misalignment;
11522 240086 : if (dataref_offset == NULL_TREE
11523 237955 : && TREE_CODE (dataref_ptr) == SSA_NAME)
11524 172148 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
11525 : misalign);
11526 240086 : align = least_bit_hwi (misalign | align);
11527 :
11528 : /* Compute IFN when LOOP_LENS or final_mask valid. */
11529 240086 : machine_mode vmode = TYPE_MODE (vectype);
11530 240086 : machine_mode new_vmode = vmode;
11531 240086 : internal_fn partial_ifn = IFN_LAST;
11532 240086 : if (loop_lens)
11533 : {
11534 0 : opt_machine_mode new_ovmode
11535 0 : = get_len_load_store_mode (vmode, true, &partial_ifn);
11536 0 : new_vmode = new_ovmode.require ();
11537 0 : unsigned factor
11538 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
11539 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11540 : vec_num, vectype, i, factor, true);
11541 : }
11542 240086 : else if (final_mask)
11543 : {
11544 682 : if (!can_vec_mask_load_store_p (vmode,
11545 682 : TYPE_MODE
11546 : (TREE_TYPE (final_mask)),
11547 : true, &partial_ifn))
11548 0 : gcc_unreachable ();
11549 : }
11550 :
11551 240086 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11552 : {
11553 0 : if (!final_len)
11554 : {
11555 : /* Pass VF value to 'len' argument of
11556 : MASK_LEN_LOAD if LOOP_LENS is invalid. */
11557 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11558 : }
11559 0 : if (!final_mask)
11560 : {
11561 : /* Pass all ones value to 'mask' argument of
11562 : MASK_LEN_LOAD if final_mask is invalid. */
11563 0 : mask_vectype = truth_type_for (vectype);
11564 0 : final_mask = build_minus_one_cst (mask_vectype);
11565 : }
11566 : }
11567 240086 : if (final_len)
11568 : {
11569 0 : signed char biasval
11570 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11571 0 : bias = build_int_cst (intQI_type_node, biasval);
11572 : }
11573 :
11574 0 : tree vec_els;
11575 :
11576 0 : if (final_len)
11577 : {
11578 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11579 0 : gcall *call;
11580 :
11581 : /* Need conversion if the vectype is punned by VnQI. */
11582 0 : els_vectype = vectype;
11583 0 : if (vmode != new_vmode)
11584 0 : els_vectype
11585 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11586 : new_vmode);
11587 0 : vec_els = vect_get_mask_load_else (maskload_elsval,
11588 : els_vectype);
11589 :
11590 0 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11591 : {
11592 0 : if (type_mode_padding_p
11593 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11594 0 : need_zeroing = true;
11595 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
11596 : 6, dataref_ptr, ptr,
11597 : final_mask, vec_els,
11598 : final_len, bias);
11599 : }
11600 : else
11601 0 : call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
11602 : dataref_ptr, ptr,
11603 : vec_els, final_len,
11604 : bias);
11605 0 : gimple_call_set_nothrow (call, true);
11606 0 : new_stmt = call;
11607 0 : data_ref = NULL_TREE;
11608 :
11609 : /* Need conversion if it's wrapped with VnQI. */
11610 0 : if (vmode != new_vmode)
11611 : {
11612 0 : tree new_vtype
11613 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11614 : new_vmode);
11615 0 : tree var = vect_get_new_ssa_name (new_vtype,
11616 : vect_simple_var);
11617 0 : gimple_set_lhs (call, var);
11618 0 : vect_finish_stmt_generation (vinfo, stmt_info, call,
11619 : gsi);
11620 0 : tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
11621 0 : new_stmt = gimple_build_assign (vec_dest,
11622 : VIEW_CONVERT_EXPR, op);
11623 : }
11624 : }
11625 240086 : else if (final_mask)
11626 : {
11627 682 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11628 682 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11629 682 : if (type_mode_padding_p
11630 682 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11631 0 : need_zeroing = true;
11632 682 : gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
11633 : dataref_ptr, ptr,
11634 : final_mask,
11635 : vec_els);
11636 682 : gimple_call_set_nothrow (call, true);
11637 682 : new_stmt = call;
11638 682 : data_ref = NULL_TREE;
11639 : }
11640 : else
11641 : {
11642 239404 : tree ltype = vectype;
11643 239404 : tree new_vtype = NULL_TREE;
11644 239404 : unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
11645 239404 : unsigned HOST_WIDE_INT dr_size
11646 239404 : = vect_get_scalar_dr_size (first_dr_info);
11647 239404 : poly_int64 off = 0;
11648 239404 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11649 1449 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
11650 239404 : unsigned int vect_align
11651 239404 : = vect_known_alignment_in_bytes (first_dr_info, vectype,
11652 239404 : off);
11653 : /* Try to use a single smaller load when we are about
11654 : to load excess elements compared to the unrolled
11655 : scalar loop. */
11656 239404 : if (known_gt ((i + 1) * nunits,
11657 : (group_size * vf - gap)))
11658 : {
11659 7093 : poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
11660 7093 : if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
11661 : nunits))
11662 : /* DR will be unused. */
11663 : ltype = NULL_TREE;
11664 2390 : else if (known_ge (vect_align,
11665 : tree_to_poly_uint64
11666 : (TYPE_SIZE_UNIT (vectype))))
11667 : /* Aligned access to excess elements is OK if
11668 : at least one element is accessed in the
11669 : scalar loop. */
11670 : ;
11671 1993 : else if (known_gt (vect_align,
11672 : ((nunits - remain) * dr_size)))
11673 : /* Aligned access to the gap area when there's
11674 : at least one element in it is OK. */
11675 : ;
11676 : else
11677 : {
11678 : /* remain should now be > 0 and < nunits. */
11679 1988 : unsigned num;
11680 1988 : if (known_ne (remain, 0u)
11681 1988 : && constant_multiple_p (nunits, remain, &num))
11682 : {
11683 1496 : tree ptype;
11684 1496 : new_vtype
11685 1496 : = vector_vector_composition_type (vectype, num,
11686 : &ptype);
11687 1496 : if (new_vtype)
11688 1496 : ltype = ptype;
11689 : }
11690 : /* Else use multiple loads or a masked load? */
11691 : /* For loop vectorization we now should have
11692 : an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11693 : set. */
11694 1988 : if (loop_vinfo)
11695 1750 : gcc_assert (new_vtype
11696 : || LOOP_VINFO_PEELING_FOR_GAPS
11697 : (loop_vinfo));
11698 : /* But still reduce the access size to the next
11699 : required power-of-two so peeling a single
11700 : scalar iteration is sufficient. */
11701 1988 : unsigned HOST_WIDE_INT cremain;
11702 1988 : if (remain.is_constant (&cremain))
11703 : {
11704 1988 : unsigned HOST_WIDE_INT cpart_size
11705 1988 : = 1 << ceil_log2 (cremain);
11706 1988 : if (known_gt (nunits, cpart_size)
11707 1988 : && constant_multiple_p (nunits, cpart_size,
11708 : &num))
11709 : {
11710 1508 : tree ptype;
11711 1508 : new_vtype
11712 3016 : = vector_vector_composition_type (vectype,
11713 1508 : num,
11714 : &ptype);
11715 1508 : if (new_vtype)
11716 1508 : ltype = ptype;
11717 : }
11718 : }
11719 : }
11720 : }
11721 239404 : tree offset = (dataref_offset ? dataref_offset
11722 237273 : : build_int_cst (ref_type, 0));
11723 239404 : if (!ltype)
11724 : ;
11725 234701 : else if (ltype != vectype
11726 234701 : && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11727 : {
11728 25 : poly_uint64 gap_offset
11729 25 : = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
11730 25 : - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
11731 25 : tree gapcst = build_int_cstu (ref_type, gap_offset);
11732 25 : offset = size_binop (PLUS_EXPR, offset, gapcst);
11733 : }
11734 234701 : if (ltype)
11735 : {
11736 234701 : data_ref = fold_build2 (MEM_REF, ltype,
11737 : dataref_ptr, offset);
11738 234701 : if (alignment_support_scheme == dr_aligned
11739 234701 : && align >= TYPE_ALIGN_UNIT (ltype))
11740 : ;
11741 : else
11742 154210 : TREE_TYPE (data_ref)
11743 308420 : = build_aligned_type (TREE_TYPE (data_ref),
11744 : align * BITS_PER_UNIT);
11745 : }
11746 234701 : if (!ltype)
11747 4703 : data_ref = build_constructor (vectype, NULL);
11748 234701 : else if (ltype != vectype)
11749 : {
11750 1508 : vect_copy_ref_info (data_ref,
11751 1508 : DR_REF (first_dr_info->dr));
11752 1508 : tree tem = make_ssa_name (ltype);
11753 1508 : new_stmt = gimple_build_assign (tem, data_ref);
11754 1508 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11755 : gsi);
11756 1508 : data_ref = NULL;
11757 1508 : vec<constructor_elt, va_gc> *v;
11758 : /* We've computed 'num' above to statically two
11759 : or via constant_multiple_p. */
11760 1508 : unsigned num
11761 1508 : = (exact_div (tree_to_poly_uint64
11762 1508 : (TYPE_SIZE_UNIT (vectype)),
11763 : tree_to_poly_uint64
11764 1508 : (TYPE_SIZE_UNIT (ltype)))
11765 1508 : .to_constant ());
11766 1508 : vec_alloc (v, num);
11767 1508 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11768 : {
11769 62 : while (--num)
11770 62 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11771 : build_zero_cst (ltype));
11772 25 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11773 : }
11774 : else
11775 : {
11776 1483 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11777 1483 : while (--num)
11778 3408 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11779 : build_zero_cst (ltype));
11780 : }
11781 1508 : gcc_assert (new_vtype != NULL_TREE);
11782 1508 : if (new_vtype == vectype)
11783 1476 : new_stmt
11784 1476 : = gimple_build_assign (vec_dest,
11785 : build_constructor (vectype, v));
11786 : else
11787 : {
11788 32 : tree new_vname = make_ssa_name (new_vtype);
11789 32 : new_stmt
11790 32 : = gimple_build_assign (new_vname,
11791 : build_constructor (new_vtype,
11792 : v));
11793 32 : vect_finish_stmt_generation (vinfo, stmt_info,
11794 : new_stmt, gsi);
11795 32 : new_stmt
11796 32 : = gimple_build_assign (vec_dest,
11797 : build1 (VIEW_CONVERT_EXPR,
11798 : vectype, new_vname));
11799 : }
11800 : }
11801 : }
11802 : break;
11803 : }
11804 0 : case dr_explicit_realign:
11805 0 : {
11806 0 : if (costing_p)
11807 : break;
11808 0 : tree ptr, bump;
11809 :
11810 0 : tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11811 :
11812 0 : if (compute_in_loop)
11813 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
11814 : gsi, &realignment_token,
11815 : dr_explicit_realign,
11816 : dataref_ptr, NULL);
11817 :
11818 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11819 0 : ptr = copy_ssa_name (dataref_ptr);
11820 : else
11821 0 : ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
11822 : // For explicit realign the target alignment should be
11823 : // known at compile time.
11824 0 : unsigned HOST_WIDE_INT align
11825 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11826 0 : new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
11827 : build_int_cst
11828 0 : (TREE_TYPE (dataref_ptr),
11829 0 : -(HOST_WIDE_INT) align));
11830 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11831 0 : data_ref = build2 (MEM_REF, vectype,
11832 : ptr, build_int_cst (ref_type, 0));
11833 0 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11834 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11835 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11836 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11837 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11838 0 : gimple_move_vops (new_stmt, stmt_info->stmt);
11839 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11840 0 : msq = new_temp;
11841 :
11842 0 : bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
11843 0 : bump = size_binop (MINUS_EXPR, bump, size_one_node);
11844 0 : ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11845 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
11846 0 : build_int_cst (TREE_TYPE (ptr),
11847 0 : -(HOST_WIDE_INT) align));
11848 0 : if (TREE_CODE (ptr) == SSA_NAME)
11849 0 : ptr = copy_ssa_name (ptr, new_stmt);
11850 : else
11851 0 : ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
11852 0 : gimple_assign_set_lhs (new_stmt, ptr);
11853 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11854 0 : data_ref = build2 (MEM_REF, vectype,
11855 : ptr, build_int_cst (ref_type, 0));
11856 0 : break;
11857 : }
11858 0 : case dr_explicit_realign_optimized:
11859 0 : {
11860 0 : if (costing_p)
11861 : break;
11862 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11863 0 : new_temp = copy_ssa_name (dataref_ptr);
11864 : else
11865 0 : new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
11866 : // We should only be doing this if we know the target
11867 : // alignment at compile time.
11868 0 : unsigned HOST_WIDE_INT align
11869 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11870 0 : new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
11871 0 : build_int_cst (TREE_TYPE (dataref_ptr),
11872 0 : -(HOST_WIDE_INT) align));
11873 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11874 0 : data_ref = build2 (MEM_REF, vectype, new_temp,
11875 : build_int_cst (ref_type, 0));
11876 0 : break;
11877 : }
11878 0 : default:
11879 0 : gcc_unreachable ();
11880 : }
11881 :
11882 : /* One common place to cost the above vect load for different
11883 : alignment support schemes. */
11884 951705 : if (costing_p)
11885 : {
11886 : /* For the prologue cost for realign,
11887 : we only need to count it once for the whole group. */
11888 711619 : bool first_stmt_info_p = first_stmt_info == stmt_info;
11889 711619 : bool add_realign_cost = first_stmt_info_p && i == 0;
11890 711619 : if (memory_access_type == VMAT_CONTIGUOUS
11891 711619 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11892 : {
11893 : /* Leave realign cases alone to keep them simple. */
11894 711619 : if (alignment_support_scheme == dr_explicit_realign_optimized
11895 : || alignment_support_scheme == dr_explicit_realign)
11896 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
11897 : alignment_support_scheme, misalignment,
11898 : add_realign_cost, &inside_cost,
11899 : &prologue_cost, cost_vec, cost_vec,
11900 : true);
11901 : else
11902 711619 : n_adjacent_loads++;
11903 : }
11904 : }
11905 : else
11906 : {
11907 240086 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11908 : /* DATA_REF is null if we've already built the statement. */
11909 240086 : if (data_ref)
11910 : {
11911 237896 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11912 237896 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11913 : }
11914 :
11915 480172 : new_temp = (need_zeroing
11916 240086 : ? make_ssa_name (vectype)
11917 240086 : : make_ssa_name (vec_dest, new_stmt));
11918 240086 : gimple_set_lhs (new_stmt, new_temp);
11919 240086 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11920 :
11921 : /* If we need to explicitly zero inactive elements emit a
11922 : VEC_COND_EXPR that does so. */
11923 240086 : if (need_zeroing)
11924 : {
11925 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11926 : vectype);
11927 :
11928 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11929 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11930 : final_mask, new_temp, vec_els);
11931 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11932 : gsi);
11933 0 : new_temp = new_temp2;
11934 : }
11935 : }
11936 :
11937 : /* 3. Handle explicit realignment if necessary/supported.
11938 : Create in loop:
11939 : vec_dest = realign_load (msq, lsq, realignment_token) */
11940 951705 : if (!costing_p
11941 240086 : && (alignment_support_scheme == dr_explicit_realign_optimized
11942 : || alignment_support_scheme == dr_explicit_realign))
11943 : {
11944 0 : lsq = gimple_assign_lhs (new_stmt);
11945 0 : if (!realignment_token)
11946 0 : realignment_token = dataref_ptr;
11947 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11948 0 : new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
11949 : lsq, realignment_token);
11950 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11951 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11952 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11953 :
11954 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
11955 : {
11956 0 : gcc_assert (phi);
11957 0 : if (i == vec_num - 1)
11958 0 : add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
11959 : UNKNOWN_LOCATION);
11960 : msq = lsq;
11961 : }
11962 : }
11963 :
11964 951705 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11965 : {
11966 5982 : if (costing_p)
11967 4533 : inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
11968 : slp_node, 0, vect_body);
11969 : else
11970 : {
11971 1449 : tree perm_mask = perm_mask_for_reverse (vectype);
11972 1449 : new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
11973 : perm_mask, stmt_info, gsi);
11974 1449 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
11975 : }
11976 : }
11977 :
11978 : /* Collect vector loads and later create their permutation in
11979 : vect_transform_slp_perm_load. */
11980 951705 : if (!costing_p && (grouped_load || ls.slp_perm))
11981 73683 : dr_chain.quick_push (new_temp);
11982 :
11983 : /* Store vector loads in the corresponding SLP_NODE. */
11984 240086 : if (!costing_p && !ls.slp_perm)
11985 166403 : slp_node->push_vec_def (new_stmt);
11986 :
11987 : /* With SLP permutation we load the gaps as well, without
11988 : we need to skip the gaps after we manage to fully load
11989 : all elements. group_gap_adj is DR_GROUP_SIZE here. */
11990 951705 : group_elt += nunits;
11991 951705 : if (!costing_p
11992 240086 : && maybe_ne (group_gap_adj, 0U)
11993 40512 : && !ls.slp_perm
11994 967326 : && known_eq (group_elt, group_size - group_gap_adj))
11995 : {
11996 12854 : poly_wide_int bump_val
11997 12854 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11998 12854 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11999 0 : bump_val = -bump_val;
12000 12854 : tree bump = wide_int_to_tree (sizetype, bump_val);
12001 12854 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
12002 : bump);
12003 12854 : group_elt = 0;
12004 12854 : }
12005 : }
12006 : /* Bump the vector pointer to account for a gap or for excess
12007 : elements loaded for a permuted SLP load. */
12008 587103 : if (!costing_p
12009 151369 : && maybe_ne (group_gap_adj, 0U)
12010 600413 : && ls.slp_perm)
12011 : {
12012 456 : poly_wide_int bump_val
12013 456 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
12014 456 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
12015 9 : bump_val = -bump_val;
12016 456 : tree bump = wide_int_to_tree (sizetype, bump_val);
12017 456 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
12018 456 : }
12019 :
12020 587103 : if (ls.slp_perm)
12021 : {
12022 : /* For SLP we know we've seen all possible uses of dr_chain so
12023 : direct vect_transform_slp_perm_load to DCE the unused parts.
12024 : ??? This is a hack to prevent compile-time issues as seen
12025 : in PR101120 and friends. */
12026 59835 : if (costing_p)
12027 : {
12028 43196 : gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
12029 43196 : if (ls.n_perms != 0)
12030 42662 : inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
12031 : slp_node, 0, vect_body);
12032 43196 : if (n_adjacent_loads > 0)
12033 43196 : n_adjacent_loads = ls.n_loads;
12034 : }
12035 : else
12036 : {
12037 16639 : unsigned n_perms2, n_loads2;
12038 16639 : bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
12039 : gsi, vf, false, &n_perms2,
12040 : &n_loads2, true);
12041 16639 : gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
12042 : }
12043 : }
12044 :
12045 587103 : if (costing_p)
12046 : {
12047 435734 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
12048 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
12049 435734 : if (n_adjacent_loads > 0)
12050 435734 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
12051 : alignment_support_scheme, misalignment, false,
12052 : &inside_cost, &prologue_cost, cost_vec, cost_vec,
12053 : true);
12054 435734 : if (dump_enabled_p ())
12055 25467 : dump_printf_loc (MSG_NOTE, vect_location,
12056 : "vect_model_load_cost: inside_cost = %u, "
12057 : "prologue_cost = %u .\n",
12058 : inside_cost, prologue_cost);
12059 435734 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
12060 435734 : slp_node->data = new vect_load_store_data (std::move (ls));
12061 : }
12062 :
12063 587103 : return true;
12064 1321655 : }
12065 :
12066 : /* vectorizable_condition.
12067 :
12068 : Check if STMT_INFO is conditional modify expression that can be vectorized.
12069 : If COST_VEC is passed, calculate costs but don't change anything,
12070 : otherwise, vectorize STMT_INFO: create a vectorized stmt using
12071 : VEC_COND_EXPR to replace it, and insert it at GSI.
12072 :
12073 : When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12074 :
12075 : Return true if STMT_INFO is vectorizable in this way. */
12076 :
12077 : static bool
12078 732898 : vectorizable_condition (vec_info *vinfo,
12079 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12080 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12081 : {
12082 732898 : tree scalar_dest = NULL_TREE;
12083 732898 : tree vec_dest = NULL_TREE;
12084 732898 : tree then_clause, else_clause;
12085 732898 : tree vec_cond_lhs = NULL_TREE;
12086 732898 : tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
12087 732898 : tree vec_compare;
12088 732898 : tree new_temp;
12089 732898 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
12090 732898 : enum vect_def_type dts[4]
12091 : = {vect_unknown_def_type, vect_unknown_def_type,
12092 : vect_unknown_def_type, vect_unknown_def_type};
12093 732898 : enum tree_code code;
12094 732898 : int i;
12095 732898 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12096 732898 : vec<tree> vec_oprnds0 = vNULL;
12097 732898 : vec<tree> vec_oprnds2 = vNULL;
12098 732898 : vec<tree> vec_oprnds3 = vNULL;
12099 732898 : tree vec_cmp_type;
12100 :
12101 732898 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12102 : return false;
12103 :
12104 : /* Is vectorizable conditional operation? */
12105 732898 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12106 419040 : if (!stmt)
12107 : return false;
12108 :
12109 419040 : code = gimple_assign_rhs_code (stmt);
12110 419040 : if (code != COND_EXPR)
12111 : return false;
12112 :
12113 35946 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
12114 35946 : vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
12115 35946 : bool nested_cycle_p = false;
12116 35946 : bool for_reduction = vect_is_reduction (stmt_info);
12117 35946 : if (for_reduction)
12118 : {
12119 648 : if (SLP_TREE_LANES (slp_node) > 1)
12120 : return false;
12121 : /* ??? With a reduction path we do not get at the reduction info from
12122 : every stmt, use the conservative default setting then. */
12123 729 : if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
12124 : {
12125 630 : vect_reduc_info reduc_info
12126 630 : = info_for_reduction (loop_vinfo, slp_node);
12127 630 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
12128 630 : nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12129 : stmt_info);
12130 : }
12131 : }
12132 : else
12133 : {
12134 35298 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12135 : return false;
12136 : }
12137 :
12138 35946 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12139 35946 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12140 :
12141 35946 : int vec_num = vect_get_num_copies (vinfo, slp_node);
12142 :
12143 35946 : slp_tree slp_cond;
12144 35946 : tree cond_expr = gimple_assign_rhs1 (stmt);
12145 35946 : gcc_assert (! COMPARISON_CLASS_P (cond_expr));
12146 35946 : if (TREE_CODE (cond_expr) != SSA_NAME
12147 35946 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond_expr))
12148 35946 : || !vect_is_simple_use (vinfo, slp_node, 0,
12149 : &slp_cond, &dts[0], &vec_cmp_type)
12150 35946 : || !vec_cmp_type
12151 71877 : || !VECTOR_BOOLEAN_TYPE_P (vec_cmp_type))
12152 : return false;
12153 :
12154 35931 : slp_tree then_slp_node, else_slp_node;
12155 35931 : if (!vect_is_simple_use (vinfo, slp_node, 1,
12156 : &then_clause, &then_slp_node, &dts[2], &vectype1))
12157 : return false;
12158 35931 : if (!vect_is_simple_use (vinfo, slp_node, 2,
12159 : &else_clause, &else_slp_node, &dts[3], &vectype2))
12160 : return false;
12161 :
12162 35931 : if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
12163 : return false;
12164 :
12165 35931 : if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
12166 : return false;
12167 :
12168 35919 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
12169 71838 : TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
12170 : return false;
12171 :
12172 : /* For conditional reductions, the "then" value needs to be the candidate
12173 : value calculated by this iteration while the "else" value needs to be
12174 : the result carried over from previous iterations. If the COND_EXPR
12175 : is the other way around, we need to swap it. */
12176 35919 : bool must_invert_cmp_result = false;
12177 35919 : if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
12178 : {
12179 732898 : must_invert_cmp_result = true;
12180 : /* ??? The vectorized operand query below doesn't allow swapping
12181 : this way for SLP. */
12182 : return false;
12183 : /* std::swap (then_clause, else_clause); */
12184 : }
12185 :
12186 35919 : if (cost_vec)
12187 : {
12188 27153 : vect_cost_for_stmt kind = vector_stmt;
12189 27153 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12190 : /* Count one reduction-like operation per vector. */
12191 : kind = vec_to_scalar;
12192 27153 : else if (!expand_vec_cond_expr_p (vectype, vec_cmp_type))
12193 : return false;
12194 :
12195 27139 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
12196 : vec_cmp_type)
12197 27139 : || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
12198 54278 : || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
12199 : {
12200 0 : if (dump_enabled_p ())
12201 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12202 : "incompatible vector types for invariants\n");
12203 : return false;
12204 : }
12205 :
12206 27139 : if (loop_vinfo && for_reduction
12207 472 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12208 : {
12209 81 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12210 : {
12211 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12212 : vectype, OPTIMIZE_FOR_SPEED))
12213 0 : vect_record_loop_len (loop_vinfo,
12214 : &LOOP_VINFO_LENS (loop_vinfo),
12215 : vec_num, vectype, 1);
12216 : else
12217 0 : vect_record_loop_mask (loop_vinfo,
12218 : &LOOP_VINFO_MASKS (loop_vinfo),
12219 : vec_num, vectype, NULL);
12220 : }
12221 : /* Extra inactive lanes should be safe for vect_nested_cycle. */
12222 81 : else if (!nested_cycle_p)
12223 : {
12224 81 : if (dump_enabled_p ())
12225 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12226 : "conditional reduction prevents the use"
12227 : " of partial vectors.\n");
12228 81 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
12229 : }
12230 : }
12231 :
12232 27139 : SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
12233 27139 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
12234 27139 : return true;
12235 : }
12236 :
12237 : /* Transform. */
12238 :
12239 : /* Handle def. */
12240 8766 : scalar_dest = gimple_assign_lhs (stmt);
12241 8766 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12242 8766 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
12243 :
12244 8766 : bool swap_cond_operands = false;
12245 :
12246 : /* See whether another part of the vectorized code applies a loop
12247 : mask to the condition, or to its inverse. */
12248 :
12249 8766 : vec_loop_masks *masks = NULL;
12250 8766 : vec_loop_lens *lens = NULL;
12251 8766 : if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12252 : {
12253 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12254 0 : lens = &LOOP_VINFO_LENS (loop_vinfo);
12255 : }
12256 8766 : else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
12257 : {
12258 3 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12259 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12260 : else
12261 : {
12262 3 : scalar_cond_masked_key cond (cond_expr, 1);
12263 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12264 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12265 : else
12266 : {
12267 3 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
12268 3 : tree_code orig_code = cond.code;
12269 3 : cond.code = invert_tree_comparison (cond.code, honor_nans);
12270 : /* Try the inverse of the current mask. We check if the
12271 : inverse mask is live and if so we generate a negate of
12272 : the current mask such that we still honor NaNs. */
12273 3 : cond.inverted_p = true;
12274 3 : cond.code = orig_code;
12275 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12276 : {
12277 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12278 0 : swap_cond_operands = true;
12279 0 : must_invert_cmp_result = true;
12280 : }
12281 : }
12282 : }
12283 : }
12284 :
12285 : /* Handle cond expr. */
12286 8766 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds2,
12287 : reduction_type != EXTRACT_LAST_REDUCTION, &vec_oprnds3);
12288 :
12289 8766 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12290 0 : vec_else_clause = else_clause;
12291 :
12292 : /* Arguments are ready. Create the new vector stmt. */
12293 20390 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
12294 : {
12295 11624 : vec_then_clause = vec_oprnds2[i];
12296 11624 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12297 11624 : vec_else_clause = vec_oprnds3[i];
12298 :
12299 11624 : if (swap_cond_operands)
12300 0 : std::swap (vec_then_clause, vec_else_clause);
12301 :
12302 11624 : vec_compare = vec_cond_lhs;
12303 :
12304 : /* If we decided to apply a loop mask to the result of the vector
12305 : comparison, AND the comparison with the mask now. Later passes
12306 : should then be able to reuse the AND results between multiple
12307 : vector statements.
12308 :
12309 : For example:
12310 : for (int i = 0; i < 100; ++i)
12311 : x[i] = y[i] ? z[i] : 10;
12312 :
12313 : results in following optimized GIMPLE:
12314 :
12315 : mask__35.8_43 = vect__4.7_41 != { 0, ... };
12316 : vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12317 : _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12318 : vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12319 : vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12320 : vect_iftmp.11_47, { 10, ... }>;
12321 :
12322 : instead of using a masked and unmasked forms of
12323 : vec != { 0, ... } (masked in the MASK_LOAD,
12324 : unmasked in the VEC_COND_EXPR). */
12325 :
12326 : /* Force vec_compare to be an SSA_NAME rather than a comparison,
12327 : in cases where that's necessary. */
12328 :
12329 11624 : tree len = NULL_TREE, bias = NULL_TREE;
12330 11624 : if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
12331 : {
12332 0 : if (!is_gimple_val (vec_compare))
12333 : {
12334 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12335 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12336 : vec_compare);
12337 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12338 0 : vec_compare = vec_compare_name;
12339 : }
12340 :
12341 0 : if (must_invert_cmp_result)
12342 : {
12343 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12344 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12345 : BIT_NOT_EXPR,
12346 : vec_compare);
12347 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12348 0 : vec_compare = vec_compare_name;
12349 : }
12350 :
12351 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12352 : vectype, OPTIMIZE_FOR_SPEED))
12353 : {
12354 0 : if (lens)
12355 : {
12356 : /* ??? Do we really want the adjusted LEN here? Isn't this
12357 : based on number of elements? */
12358 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens,
12359 : vec_num, vectype, i, 1, true);
12360 0 : signed char biasval
12361 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
12362 0 : bias = build_int_cst (intQI_type_node, biasval);
12363 : }
12364 : else
12365 : {
12366 0 : len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
12367 0 : bias = build_int_cst (intQI_type_node, 0);
12368 : }
12369 : }
12370 0 : if (masks)
12371 : {
12372 0 : tree loop_mask
12373 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12374 : vectype, i);
12375 0 : tree tmp2 = make_ssa_name (vec_cmp_type);
12376 0 : gassign *g
12377 0 : = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
12378 : loop_mask);
12379 0 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
12380 0 : vec_compare = tmp2;
12381 : }
12382 : }
12383 :
12384 0 : gimple *new_stmt;
12385 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12386 : {
12387 0 : gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
12388 0 : tree lhs = gimple_get_lhs (old_stmt);
12389 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12390 0 : lhs = copy_ssa_name (lhs);
12391 0 : if (len)
12392 0 : new_stmt = gimple_build_call_internal
12393 0 : (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
12394 : vec_then_clause, len, bias);
12395 : else
12396 0 : new_stmt = gimple_build_call_internal
12397 0 : (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
12398 : vec_then_clause);
12399 0 : gimple_call_set_lhs (new_stmt, lhs);
12400 0 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12401 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12402 : {
12403 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12404 0 : vec_else_clause = lhs;
12405 : }
12406 0 : else if (old_stmt == gsi_stmt (*gsi))
12407 0 : vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
12408 : else
12409 : {
12410 : /* In this case we're moving the definition to later in the
12411 : block. That doesn't matter because the only uses of the
12412 : lhs are in phi statements. */
12413 0 : gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
12414 0 : gsi_remove (&old_gsi, true);
12415 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12416 : }
12417 : }
12418 : else
12419 : {
12420 11624 : new_temp = make_ssa_name (vec_dest);
12421 11624 : new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
12422 : vec_then_clause, vec_else_clause);
12423 11624 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12424 : }
12425 11624 : slp_node->push_vec_def (new_stmt);
12426 : }
12427 :
12428 8766 : vec_oprnds0.release ();
12429 8766 : vec_oprnds2.release ();
12430 8766 : vec_oprnds3.release ();
12431 :
12432 8766 : return true;
12433 : }
12434 :
12435 : /* Helper of vectorizable_comparison.
12436 :
12437 : Check if STMT_INFO is comparison expression CODE that can be vectorized.
12438 : If COST_VEC is passed, calculate costs but don't change anything,
12439 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12440 : it at GSI.
12441 :
12442 : Return true if STMT_INFO is vectorizable in this way. */
12443 :
12444 : static bool
12445 391633 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
12446 : stmt_vec_info stmt_info, tree_code code,
12447 : gimple_stmt_iterator *gsi,
12448 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12449 : {
12450 391633 : tree lhs, rhs1, rhs2;
12451 391633 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12452 391633 : tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
12453 391633 : tree new_temp;
12454 391633 : enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
12455 391633 : poly_uint64 nunits;
12456 391633 : enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12457 391633 : int i;
12458 391633 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12459 391633 : vec<tree> vec_oprnds0 = vNULL;
12460 391633 : vec<tree> vec_oprnds1 = vNULL;
12461 391633 : tree mask_type;
12462 391633 : tree mask = NULL_TREE;
12463 :
12464 391633 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12465 : return false;
12466 :
12467 391633 : if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
12468 : return false;
12469 :
12470 174163 : mask_type = vectype;
12471 174163 : nunits = TYPE_VECTOR_SUBPARTS (vectype);
12472 :
12473 174163 : if (TREE_CODE_CLASS (code) != tcc_comparison)
12474 : return false;
12475 :
12476 172213 : slp_tree slp_rhs1, slp_rhs2;
12477 172213 : if (!vect_is_simple_use (vinfo, slp_node,
12478 : 0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
12479 : return false;
12480 :
12481 172213 : if (!vect_is_simple_use (vinfo, slp_node,
12482 : 1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
12483 : return false;
12484 :
12485 134456 : if (vectype1 && vectype2
12486 249454 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12487 77241 : TYPE_VECTOR_SUBPARTS (vectype2)))
12488 16 : return false;
12489 :
12490 172197 : vectype = vectype1 ? vectype1 : vectype2;
12491 :
12492 : /* Invariant comparison. */
12493 172197 : if (!vectype)
12494 : {
12495 33235 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
12496 33235 : if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
12497 : return false;
12498 : }
12499 138962 : else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
12500 : return false;
12501 :
12502 : /* Can't compare mask and non-mask types. */
12503 134440 : if (vectype1 && vectype2
12504 403236 : && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
12505 : return false;
12506 :
12507 : /* We cannot compare non-mode precision _BitInt types. Unlike bool
12508 : or bit-precision INTEGER_TYPE the padding bit values are target
12509 : dependent and possibly undefined. */
12510 172185 : if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE
12511 172185 : && !type_has_mode_precision_p (TREE_TYPE (rhs1)))
12512 : return false;
12513 :
12514 : /* Boolean values may have another representation in vectors
12515 : and therefore we prefer bit operations over comparison for
12516 : them (which also works for scalar masks). We store opcodes
12517 : to use in bitop1 and bitop2. Statement is vectorized as
12518 : BITOP2 (rhs1 BITOP1 rhs2) or
12519 : rhs1 BITOP2 (BITOP1 rhs2)
12520 : depending on bitop1 and bitop2 arity. */
12521 172099 : bool swap_p = false;
12522 172099 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
12523 : {
12524 765 : if (code == GT_EXPR)
12525 : {
12526 : bitop1 = BIT_NOT_EXPR;
12527 : bitop2 = BIT_AND_EXPR;
12528 : }
12529 : else if (code == GE_EXPR)
12530 : {
12531 : bitop1 = BIT_NOT_EXPR;
12532 : bitop2 = BIT_IOR_EXPR;
12533 : }
12534 : else if (code == LT_EXPR)
12535 : {
12536 : bitop1 = BIT_NOT_EXPR;
12537 : bitop2 = BIT_AND_EXPR;
12538 : swap_p = true;
12539 : }
12540 : else if (code == LE_EXPR)
12541 : {
12542 : bitop1 = BIT_NOT_EXPR;
12543 : bitop2 = BIT_IOR_EXPR;
12544 : swap_p = true;
12545 : }
12546 : else
12547 : {
12548 : bitop1 = BIT_XOR_EXPR;
12549 : if (code == EQ_EXPR)
12550 : bitop2 = BIT_NOT_EXPR;
12551 : }
12552 : }
12553 :
12554 172099 : if (cost_vec)
12555 : {
12556 159365 : if (bitop1 == NOP_EXPR)
12557 : {
12558 158749 : if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
12559 : return false;
12560 : }
12561 : else
12562 : {
12563 616 : machine_mode mode = TYPE_MODE (vectype);
12564 616 : optab optab;
12565 :
12566 616 : optab = optab_for_tree_code (bitop1, vectype, optab_default);
12567 616 : if (!optab || !can_implement_p (optab, mode))
12568 : return false;
12569 :
12570 616 : if (bitop2 != NOP_EXPR)
12571 : {
12572 97 : optab = optab_for_tree_code (bitop2, vectype, optab_default);
12573 97 : if (!optab || !can_implement_p (optab, mode))
12574 : return false;
12575 : }
12576 : }
12577 :
12578 : /* Put types on constant and invariant SLP children. */
12579 149083 : if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
12580 149083 : || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
12581 : {
12582 2 : if (dump_enabled_p ())
12583 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12584 : "incompatible vector types for invariants\n");
12585 : return false;
12586 : }
12587 :
12588 149081 : vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
12589 : slp_node, cost_vec);
12590 149081 : return true;
12591 : }
12592 :
12593 : /* Transform. */
12594 :
12595 : /* Handle def. */
12596 12734 : lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
12597 12734 : if (lhs)
12598 12734 : mask = vect_create_destination_var (lhs, mask_type);
12599 :
12600 12734 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds1);
12601 12734 : if (swap_p)
12602 60 : std::swap (vec_oprnds0, vec_oprnds1);
12603 :
12604 : /* Arguments are ready. Create the new vector stmt. */
12605 32001 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
12606 : {
12607 19267 : gimple *new_stmt;
12608 19267 : vec_rhs2 = vec_oprnds1[i];
12609 :
12610 19267 : if (lhs)
12611 19267 : new_temp = make_ssa_name (mask);
12612 : else
12613 0 : new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
12614 19267 : if (bitop1 == NOP_EXPR)
12615 : {
12616 19108 : new_stmt = gimple_build_assign (new_temp, code,
12617 : vec_rhs1, vec_rhs2);
12618 19108 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12619 : }
12620 : else
12621 : {
12622 159 : if (bitop1 == BIT_NOT_EXPR)
12623 86 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
12624 : else
12625 73 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
12626 : vec_rhs2);
12627 159 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12628 159 : if (bitop2 != NOP_EXPR)
12629 : {
12630 86 : tree res = make_ssa_name (mask);
12631 86 : if (bitop2 == BIT_NOT_EXPR)
12632 0 : new_stmt = gimple_build_assign (res, bitop2, new_temp);
12633 : else
12634 86 : new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
12635 : new_temp);
12636 86 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12637 : }
12638 : }
12639 19267 : slp_node->push_vec_def (new_stmt);
12640 : }
12641 :
12642 12734 : vec_oprnds0.release ();
12643 12734 : vec_oprnds1.release ();
12644 :
12645 12734 : return true;
12646 : }
12647 :
12648 : /* vectorizable_comparison.
12649 :
12650 : Check if STMT_INFO is comparison expression that can be vectorized.
12651 : If COST_VEC is passed, calculate costs but don't change anything,
12652 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12653 : it at GSI.
12654 :
12655 : Return true if STMT_INFO is vectorizable in this way. */
12656 :
12657 : static bool
12658 709727 : vectorizable_comparison (vec_info *vinfo,
12659 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12660 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12661 : {
12662 709727 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12663 :
12664 709727 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12665 : return false;
12666 :
12667 709727 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12668 : return false;
12669 :
12670 462416 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12671 388989 : if (!stmt)
12672 : return false;
12673 :
12674 388989 : enum tree_code code = gimple_assign_rhs_code (stmt);
12675 388989 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12676 388989 : if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
12677 : slp_node, cost_vec))
12678 : return false;
12679 :
12680 159171 : if (cost_vec)
12681 146437 : SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
12682 :
12683 : return true;
12684 : }
12685 :
12686 : /* Check to see if the target supports any of the compare and branch optabs for
12687 : vectors with MODE as these would be required when expanding. */
12688 : static bool
12689 66418 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
12690 : {
12691 66418 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12692 66418 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12693 :
12694 : /* The vectorizer only produces vec_cbranch_any_optab directly. So only
12695 : check for support for that or vec_cbranch_any_optab when masked.
12696 : We can't produce vcond_cbranch_any directly from the vectorizer as we
12697 : want to keep gimple_cond as the GIMPLE representation. But we'll fold
12698 : it in expand. For that reason we require a backend to support the
12699 : unconditional vector cbranch optab if they support the conditional one,
12700 : which is just an optimization on the unconditional one. */
12701 66418 : if (masked_loop_p
12702 66418 : && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
12703 : != CODE_FOR_nothing)
12704 : return true;
12705 66418 : else if (len_loop_p
12706 66418 : && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
12707 : != CODE_FOR_nothing)
12708 : return true;
12709 66418 : else if (!masked_loop_p && !len_loop_p
12710 132836 : && direct_optab_handler (vec_cbranch_any_optab, mode)
12711 : != CODE_FOR_nothing)
12712 : return true;
12713 :
12714 : /* The target can implement cbranch to distinguish between boolean vector
12715 : types and data types if they don't have a different mode for both. */
12716 66418 : return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
12717 : }
12718 :
12719 : /* Determine the type to use for early break vectorization's scalar IV. If
12720 : no type is possible return false. */
12721 :
12722 : static bool
12723 2644 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
12724 : {
12725 : /* Check if we have a usable scalar IV type for vectorization. */
12726 2644 : tree iters_vf_type = sizetype;
12727 2644 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
12728 : {
12729 : /* Find the type with the minimum precision we can use
12730 : for the scalar IV. */
12731 2421 : tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
12732 :
12733 : /* Work out how many bits we need to represent the limit. */
12734 2421 : unsigned int min_ni_width
12735 2421 : = vect_min_prec_for_max_niters (loop_vinfo, 1);
12736 :
12737 : /* Check if we're using PFA, if so we need a signed IV and an
12738 : extra bit for the sign. */
12739 2421 : if (TYPE_UNSIGNED (cand_type)
12740 2421 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
12741 3992 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12742 159 : min_ni_width += 1;
12743 :
12744 2421 : if (TYPE_PRECISION (cand_type) >= min_ni_width)
12745 2348 : iters_vf_type = unsigned_type_for (cand_type);
12746 : else
12747 : {
12748 73 : opt_scalar_int_mode cmp_mode_iter;
12749 73 : tree iv_type = NULL_TREE;
12750 357 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
12751 : {
12752 357 : auto cmp_mode = cmp_mode_iter.require ();
12753 357 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
12754 357 : if (cmp_bits >= min_ni_width
12755 357 : && targetm.scalar_mode_supported_p (cmp_mode))
12756 : {
12757 73 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
12758 73 : if (iv_type)
12759 : break;
12760 : }
12761 : }
12762 :
12763 73 : if (!iv_type)
12764 : {
12765 0 : if (dump_enabled_p ())
12766 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12767 : "can't vectorize early exit because the "
12768 : "target doesn't support a scalar type wide "
12769 : "wide enough to hold niters.\n");
12770 0 : return false;
12771 : }
12772 73 : iters_vf_type = iv_type;
12773 : }
12774 : }
12775 :
12776 2644 : LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
12777 2644 : return true;
12778 : }
12779 :
12780 : /* Check to see if the current early break given in STMT_INFO is valid for
12781 : vectorization. */
12782 :
12783 : bool
12784 265942 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
12785 : gimple_stmt_iterator *gsi,
12786 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12787 : {
12788 265942 : if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
12789 : return false;
12790 :
12791 68014 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
12792 : return false;
12793 :
12794 68014 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
12795 : return false;
12796 :
12797 68014 : DUMP_VECT_SCOPE ("vectorizable_early_exit");
12798 :
12799 68014 : auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
12800 :
12801 : /* For SLP we don't want to use the type of the operands of the SLP node, when
12802 : vectorizing using SLP slp_node will be the children of the gcond and we
12803 : want to use the type of the direct children which since the gcond is root
12804 : will be the current node, rather than a child node as vect_is_simple_use
12805 : assumes. */
12806 68014 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12807 68014 : if (!vectype)
12808 : return false;
12809 :
12810 68014 : machine_mode mode = TYPE_MODE (vectype);
12811 68014 : int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
12812 :
12813 68014 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
12814 68014 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
12815 68014 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12816 68014 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12817 :
12818 : /* Now build the new conditional. Pattern gimple_conds get dropped during
12819 : codegen so we must replace the original insn. */
12820 68014 : gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
12821 68014 : gcond *cond_stmt = as_a <gcond *>(orig_stmt);
12822 :
12823 68014 : tree vectype_out = vectype;
12824 68014 : auto bb = gimple_bb (cond_stmt);
12825 68014 : edge exit_true_edge = EDGE_SUCC (bb, 0);
12826 68014 : if (exit_true_edge->flags & EDGE_FALSE_VALUE)
12827 825 : exit_true_edge = EDGE_SUCC (bb, 1);
12828 68014 : gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
12829 :
12830 : /* When vectorizing we assume that if the branch edge is taken that we're
12831 : exiting the loop. This is not however always the case as the compiler will
12832 : rewrite conditions to always be a comparison against 0. To do this it
12833 : sometimes flips the edges. This is fine for scalar, but for vector we
12834 : then have to negate the result of the test, as we're still assuming that if
12835 : you take the branch edge that we found the exit condition. i.e. we need to
12836 : know whether we are generating a `forall` or an `exist` condition. */
12837 136028 : bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12838 68014 : exit_true_edge->dest);
12839 :
12840 : /* See if we support ADDHN and use that for the reduction. */
12841 68014 : internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
12842 68014 : bool addhn_supported_p
12843 68014 : = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
12844 68014 : tree narrow_type = NULL_TREE;
12845 68014 : if (addhn_supported_p)
12846 : {
12847 : /* Calculate the narrowing type for the result. */
12848 0 : auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
12849 0 : auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
12850 0 : tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
12851 0 : tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
12852 0 : narrow_type = truth_type_for (tmp_type);
12853 :
12854 0 : if (!supports_vector_compare_and_branch (loop_vinfo,
12855 0 : TYPE_MODE (narrow_type)))
12856 : {
12857 0 : if (dump_enabled_p ())
12858 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12859 : "can't use ADDHN reduction because cbranch for "
12860 : "the narrowed type is not supported by the "
12861 : "target.\n");
12862 : addhn_supported_p = false;
12863 : }
12864 : }
12865 :
12866 : /* Analyze only. */
12867 68014 : if (cost_vec)
12868 : {
12869 66418 : if (!addhn_supported_p
12870 66418 : && !supports_vector_compare_and_branch (loop_vinfo, mode))
12871 : {
12872 63774 : if (dump_enabled_p ())
12873 777 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12874 : "can't vectorize early exit because the "
12875 : "target doesn't support flag setting vector "
12876 : "comparisons.\n");
12877 : return false;
12878 : }
12879 :
12880 2644 : if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
12881 : slp_node, cost_vec))
12882 : return false;
12883 :
12884 2644 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12885 : {
12886 1571 : if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
12887 : OPTIMIZE_FOR_SPEED))
12888 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
12889 : else
12890 1571 : vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
12891 : }
12892 :
12893 2644 : if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
12894 : return false;
12895 :
12896 : return true;
12897 : }
12898 :
12899 : /* Transform. */
12900 :
12901 1596 : tree new_temp = NULL_TREE;
12902 1596 : gimple *new_stmt = NULL;
12903 :
12904 1596 : if (dump_enabled_p ())
12905 404 : dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
12906 :
12907 : /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
12908 : roots and so by the time we get to them we have already codegened the SLP tree
12909 : and so we shouldn't try to do so again. The arguments have already been
12910 : vectorized. It's not very clean to do this here, But the masking code below is
12911 : complex and this keeps it all in one place to ease fixes and backports. Once we
12912 : drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
12913 :
12914 1596 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
12915 1596 : basic_block cond_bb = gimple_bb (stmt);
12916 1596 : gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
12917 :
12918 1596 : auto_vec<tree> stmts;
12919 1596 : stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
12920 :
12921 : /* If we're comparing against a previous forall we need to negate the results
12922 : before we do the final comparison or reduction. */
12923 1596 : if (flipped)
12924 : {
12925 : /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
12926 : if (any(~mask)) by negating the masks and flipping the branches.
12927 :
12928 : 1. For unmasked loops we simply reduce the ~mask.
12929 : 2. For masked loops we reduce (~mask & loop_mask) which is the same as
12930 : doing (mask & loop_mask) ^ loop_mask. */
12931 319 : for (unsigned i = 0; i < stmts.length (); i++)
12932 : {
12933 191 : tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
12934 191 : auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
12935 191 : vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
12936 : &cond_gsi);
12937 191 : stmts[i] = inv_lhs;
12938 : }
12939 :
12940 128 : EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12941 128 : EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12942 : }
12943 :
12944 : /* Determine if we need to reduce the final value. */
12945 1596 : if (stmts.length () > 1)
12946 : {
12947 : /* We build the reductions in a way to maintain as much parallelism as
12948 : possible. */
12949 147 : auto_vec<tree> workset (stmts.length ());
12950 :
12951 : /* Mask the statements as we queue them up. Normally we loop over
12952 : vec_num, but since we inspect the exact results of vectorization
12953 : we don't need to and instead can just use the stmts themselves. */
12954 147 : if (masked_loop_p)
12955 0 : for (unsigned i = 0; i < stmts.length (); i++)
12956 : {
12957 0 : tree stmt_mask
12958 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12959 : vectype, i);
12960 0 : stmt_mask
12961 0 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
12962 0 : stmts[i], &cond_gsi);
12963 0 : workset.quick_push (stmt_mask);
12964 : }
12965 147 : else if (len_loop_p)
12966 147 : for (unsigned i = 0; i < stmts.length (); i++)
12967 : {
12968 0 : tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
12969 : lens, vec_num,
12970 0 : vectype, stmts[i], i, 1);
12971 :
12972 0 : workset.quick_push (len_mask);
12973 : }
12974 : else
12975 147 : workset.splice (stmts);
12976 :
12977 442 : while (workset.length () > 1)
12978 : {
12979 295 : tree arg0 = workset.pop ();
12980 295 : tree arg1 = workset.pop ();
12981 295 : if (addhn_supported_p && workset.length () == 0)
12982 : {
12983 0 : new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
12984 0 : vectype_out = narrow_type;
12985 0 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12986 0 : gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
12987 0 : gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
12988 : }
12989 : else
12990 : {
12991 295 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12992 295 : new_stmt
12993 295 : = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
12994 : }
12995 295 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
12996 : &cond_gsi);
12997 295 : workset.quick_insert (0, new_temp);
12998 : }
12999 147 : }
13000 : else
13001 : {
13002 1449 : new_temp = stmts[0];
13003 1449 : if (masked_loop_p)
13004 : {
13005 2 : tree mask
13006 2 : = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
13007 2 : new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
13008 : new_temp, &cond_gsi);
13009 : }
13010 1447 : else if (len_loop_p)
13011 0 : new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
13012 : 1, vectype, new_temp, 0, 1);
13013 : }
13014 :
13015 1596 : gcc_assert (new_temp);
13016 :
13017 1596 : tree cst = build_zero_cst (vectype_out);
13018 1596 : gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
13019 1596 : update_stmt (orig_stmt);
13020 :
13021 : /* ??? */
13022 1596 : SLP_TREE_VEC_DEFS (slp_node).truncate (0);
13023 :
13024 1596 : return true;
13025 1596 : }
13026 :
13027 : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13028 : can handle all live statements in the node. Otherwise return true
13029 : if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13030 : VEC_STMT_P is as for vectorizable_live_operation. */
13031 :
13032 : static bool
13033 1335356 : can_vectorize_live_stmts (vec_info *vinfo,
13034 : slp_tree slp_node, slp_instance slp_node_instance,
13035 : bool vec_stmt_p,
13036 : stmt_vector_for_cost *cost_vec)
13037 : {
13038 1335356 : stmt_vec_info slp_stmt_info;
13039 1335356 : unsigned int i;
13040 2813774 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
13041 : {
13042 1478418 : if (slp_stmt_info
13043 1462164 : && STMT_VINFO_LIVE_P (slp_stmt_info)
13044 1618432 : && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13045 : slp_node_instance, i,
13046 : vec_stmt_p, cost_vec))
13047 : return false;
13048 : }
13049 :
13050 : return true;
13051 : }
13052 :
13053 : /* Make sure the statement is vectorizable. */
13054 :
13055 : opt_result
13056 2660827 : vect_analyze_stmt (vec_info *vinfo,
13057 : slp_tree node, slp_instance node_instance,
13058 : stmt_vector_for_cost *cost_vec)
13059 : {
13060 2660827 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
13061 2660827 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
13062 2660827 : enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
13063 2660827 : bool ok;
13064 :
13065 2660827 : if (dump_enabled_p ())
13066 105989 : dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
13067 : stmt_info->stmt);
13068 :
13069 5020684 : if (gimple_has_volatile_ops (stmt_info->stmt))
13070 : {
13071 : /* ??? This shouldn't really happen, volatile stmts should
13072 : not end up in the SLP graph. */
13073 0 : return opt_result::failure_at (stmt_info->stmt,
13074 : "not vectorized:"
13075 : " stmt has volatile operands: %G\n",
13076 : stmt_info->stmt);
13077 : }
13078 :
13079 : /* Skip stmts that do not need to be vectorized. */
13080 2660827 : if (!STMT_VINFO_RELEVANT_P (stmt_info)
13081 0 : && !STMT_VINFO_LIVE_P (stmt_info))
13082 : {
13083 0 : if (dump_enabled_p ())
13084 0 : dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
13085 :
13086 : /* ??? This shouldn't really happen, irrelevant stmts should
13087 : not end up in the SLP graph. */
13088 0 : return opt_result::failure_at (stmt_info->stmt,
13089 : "not vectorized:"
13090 : " irrelevant stmt as SLP node %p "
13091 : "representative.\n",
13092 : (void *)node);
13093 : }
13094 :
13095 2660827 : switch (STMT_VINFO_DEF_TYPE (stmt_info))
13096 : {
13097 : case vect_internal_def:
13098 : case vect_condition_def:
13099 : break;
13100 :
13101 85048 : case vect_reduction_def:
13102 85048 : case vect_nested_cycle:
13103 85048 : gcc_assert (!bb_vinfo
13104 : && (relevance == vect_used_in_outer
13105 : || relevance == vect_used_in_outer_by_reduction
13106 : || relevance == vect_used_by_reduction
13107 : || relevance == vect_unused_in_scope
13108 : || relevance == vect_used_only_live));
13109 : break;
13110 :
13111 322 : case vect_double_reduction_def:
13112 322 : gcc_assert (!bb_vinfo && node);
13113 : break;
13114 :
13115 161935 : case vect_induction_def:
13116 161935 : case vect_first_order_recurrence:
13117 161935 : gcc_assert (!bb_vinfo);
13118 : break;
13119 :
13120 0 : case vect_constant_def:
13121 0 : case vect_external_def:
13122 0 : case vect_unknown_def_type:
13123 0 : default:
13124 0 : gcc_unreachable ();
13125 : }
13126 :
13127 2660827 : tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
13128 2660827 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13129 :
13130 2660827 : if (STMT_VINFO_RELEVANT_P (stmt_info))
13131 : {
13132 2660827 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
13133 2660827 : gcc_assert (SLP_TREE_VECTYPE (node)
13134 : || gimple_code (stmt_info->stmt) == GIMPLE_COND
13135 : || (call && gimple_call_lhs (call) == NULL_TREE));
13136 : }
13137 :
13138 2660827 : ok = true;
13139 2660827 : if (bb_vinfo
13140 1533284 : || (STMT_VINFO_RELEVANT_P (stmt_info)
13141 0 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
13142 : /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13143 : -mveclibabi= takes preference over library functions with
13144 : the simd attribute. */
13145 2660827 : ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
13146 2653325 : || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
13147 : cost_vec)
13148 2652836 : || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
13149 2566229 : || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
13150 2137310 : || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
13151 2065948 : || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
13152 1604163 : || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
13153 769875 : || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
13154 724132 : || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
13155 696993 : || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
13156 550556 : || (bb_vinfo
13157 142364 : && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
13158 3150499 : || (is_a <loop_vec_info> (vinfo)
13159 408192 : && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
13160 : stmt_info, node, cost_vec)
13161 407452 : || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
13162 : stmt_info,
13163 : node, node_instance, cost_vec)
13164 325017 : || vectorizable_induction (as_a <loop_vec_info> (vinfo),
13165 : stmt_info, node, cost_vec)
13166 198989 : || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
13167 : stmt_info, node)
13168 198197 : || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13169 : stmt_info, node, cost_vec)
13170 197928 : || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13171 : stmt_info, NULL, node,
13172 : cost_vec))));
13173 :
13174 2660827 : STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
13175 :
13176 2381419 : if (!ok)
13177 279408 : return opt_result::failure_at (stmt_info->stmt,
13178 : "not vectorized:"
13179 : " relevant stmt not supported: %G",
13180 : stmt_info->stmt);
13181 :
13182 : /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13183 : need extra handling, except for vectorizable reductions. */
13184 2381419 : if (!bb_vinfo
13185 1335356 : && (SLP_TREE_TYPE (node) != lc_phi_info_type
13186 792 : || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
13187 1335356 : && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
13188 3716775 : && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
13189 : node, node_instance,
13190 : false, cost_vec))
13191 0 : return opt_result::failure_at (stmt_info->stmt,
13192 : "not vectorized:"
13193 : " live stmt not supported: %G",
13194 : stmt_info->stmt);
13195 :
13196 2381419 : return opt_result::success ();
13197 : }
13198 :
13199 :
13200 : /* Function vect_transform_stmt.
13201 :
13202 : Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13203 :
13204 : void
13205 977256 : vect_transform_stmt (vec_info *vinfo,
13206 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
13207 : slp_tree slp_node, slp_instance slp_node_instance)
13208 : {
13209 977256 : bool done;
13210 :
13211 977256 : gcc_assert (slp_node);
13212 :
13213 977256 : if (stmt_info)
13214 960410 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13215 :
13216 977256 : switch (SLP_TREE_TYPE (slp_node))
13217 : {
13218 24025 : case type_demotion_vec_info_type:
13219 24025 : case type_promotion_vec_info_type:
13220 24025 : case type_conversion_vec_info_type:
13221 24025 : done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
13222 24025 : gcc_assert (done);
13223 : break;
13224 :
13225 16839 : case induc_vec_info_type:
13226 16839 : done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
13227 : stmt_info, slp_node, NULL);
13228 16839 : gcc_assert (done);
13229 : break;
13230 :
13231 9659 : case shift_vec_info_type:
13232 9659 : done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
13233 9659 : gcc_assert (done);
13234 : break;
13235 :
13236 121018 : case op_vec_info_type:
13237 121018 : done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
13238 121018 : gcc_assert (done);
13239 : break;
13240 :
13241 16329 : case assignment_vec_info_type:
13242 16329 : done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
13243 16329 : gcc_assert (done);
13244 : break;
13245 :
13246 156487 : case load_vec_info_type:
13247 156487 : done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
13248 156487 : gcc_assert (done);
13249 : break;
13250 :
13251 550279 : case store_vec_info_type:
13252 550279 : done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
13253 550279 : gcc_assert (done);
13254 : break;
13255 :
13256 8766 : case condition_vec_info_type:
13257 8766 : done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
13258 8766 : gcc_assert (done);
13259 : break;
13260 :
13261 12734 : case comparison_vec_info_type:
13262 12734 : done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
13263 12734 : gcc_assert (done);
13264 : break;
13265 :
13266 4328 : case call_vec_info_type:
13267 4328 : done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
13268 4328 : break;
13269 :
13270 370 : case call_simd_clone_vec_info_type:
13271 370 : done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
13272 : slp_node, NULL);
13273 370 : break;
13274 :
13275 2640 : case reduc_vec_info_type:
13276 2640 : done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
13277 : gsi, slp_node);
13278 2640 : gcc_assert (done);
13279 : break;
13280 :
13281 23620 : case cycle_phi_info_type:
13282 23620 : done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
13283 : slp_node, slp_node_instance);
13284 23620 : gcc_assert (done);
13285 : break;
13286 :
13287 516 : case lc_phi_info_type:
13288 516 : done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
13289 : stmt_info, slp_node);
13290 516 : gcc_assert (done);
13291 : break;
13292 :
13293 43 : case recurr_info_type:
13294 43 : done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13295 : stmt_info, slp_node, NULL);
13296 43 : gcc_assert (done);
13297 : break;
13298 :
13299 12757 : case phi_info_type:
13300 12757 : done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
13301 : stmt_info, slp_node, NULL);
13302 12757 : gcc_assert (done);
13303 : break;
13304 :
13305 0 : case loop_exit_ctrl_vec_info_type:
13306 0 : done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13307 : stmt_info, gsi, slp_node, NULL);
13308 0 : gcc_assert (done);
13309 : break;
13310 :
13311 16846 : case permute_info_type:
13312 16846 : done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
13313 16846 : gcc_assert (done);
13314 : break;
13315 :
13316 0 : default:
13317 0 : if (!STMT_VINFO_LIVE_P (stmt_info))
13318 : {
13319 0 : if (dump_enabled_p ())
13320 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13321 : "stmt not supported.\n");
13322 0 : gcc_unreachable ();
13323 : }
13324 977256 : done = true;
13325 : }
13326 :
13327 977256 : if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
13328 426977 : && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
13329 : {
13330 : /* Handle stmts whose DEF is used outside the loop-nest that is
13331 : being vectorized. */
13332 536316 : for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
13333 : {
13334 40447 : stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
13335 40447 : done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13336 : slp_node_instance, lane,
13337 : true, NULL);
13338 40447 : gcc_assert (done);
13339 : }
13340 : }
13341 977256 : }
13342 :
13343 :
13344 : /* Remove a group of stores (for SLP or interleaving), free their
13345 : stmt_vec_info. */
13346 :
13347 : void
13348 0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
13349 : {
13350 0 : stmt_vec_info next_stmt_info = first_stmt_info;
13351 :
13352 0 : while (next_stmt_info)
13353 : {
13354 0 : stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
13355 0 : next_stmt_info = vect_orig_stmt (next_stmt_info);
13356 : /* Free the attached stmt_vec_info and remove the stmt. */
13357 0 : vinfo->remove_stmt (next_stmt_info);
13358 0 : next_stmt_info = tmp;
13359 : }
13360 0 : }
13361 :
13362 : /* If NUNITS is nonzero, return a vector type that contains NUNITS
13363 : elements of type SCALAR_TYPE, or null if the target doesn't support
13364 : such a type.
13365 :
13366 : If NUNITS is zero, return a vector type that contains elements of
13367 : type SCALAR_TYPE, choosing whichever vector size the target prefers.
13368 :
13369 : If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13370 : for this vectorization region and want to "autodetect" the best choice.
13371 : Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13372 : and we want the new type to be interoperable with it. PREVAILING_MODE
13373 : in this case can be a scalar integer mode or a vector mode; when it
13374 : is a vector mode, the function acts like a tree-level version of
13375 : related_vector_mode. */
13376 :
13377 : tree
13378 31268853 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
13379 : tree scalar_type, poly_uint64 nunits)
13380 : {
13381 31268853 : tree orig_scalar_type = scalar_type;
13382 31268853 : scalar_mode inner_mode;
13383 31268853 : machine_mode simd_mode;
13384 31268853 : tree vectype;
13385 :
13386 31268853 : if ((!INTEGRAL_TYPE_P (scalar_type)
13387 10269452 : && !POINTER_TYPE_P (scalar_type)
13388 2162732 : && !SCALAR_FLOAT_TYPE_P (scalar_type))
13389 40987634 : || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
13390 1612164 : && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
13391 : return NULL_TREE;
13392 :
13393 30714855 : unsigned int nbytes = GET_MODE_SIZE (inner_mode);
13394 :
13395 : /* Interoperability between modes requires one to be a constant multiple
13396 : of the other, so that the number of vectors required for each operation
13397 : is a compile-time constant. */
13398 30714855 : if (prevailing_mode != VOIDmode
13399 29532484 : && !constant_multiple_p (nunits * nbytes,
13400 29532484 : GET_MODE_SIZE (prevailing_mode))
13401 31956971 : && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
13402 1242116 : nunits * nbytes))
13403 : return NULL_TREE;
13404 :
13405 : /* For vector types of elements whose mode precision doesn't
13406 : match their types precision we use a element type of mode
13407 : precision. The vectorization routines will have to make sure
13408 : they support the proper result truncation/extension.
13409 : We also make sure to build vector types with INTEGER_TYPE
13410 : component type only. */
13411 30714855 : if (INTEGRAL_TYPE_P (scalar_type)
13412 51714153 : && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
13413 19776951 : || TREE_CODE (scalar_type) != INTEGER_TYPE))
13414 1444325 : scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
13415 1444325 : TYPE_UNSIGNED (scalar_type));
13416 :
13417 : /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13418 : When the component mode passes the above test simply use a type
13419 : corresponding to that mode. The theory is that any use that
13420 : would cause problems with this will disable vectorization anyway. */
13421 29270530 : else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
13422 : && !INTEGRAL_TYPE_P (scalar_type))
13423 8106720 : scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
13424 :
13425 : /* We can't build a vector type of elements with alignment bigger than
13426 : their size. */
13427 21163810 : else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
13428 417468 : scalar_type = lang_hooks.types.type_for_mode (inner_mode,
13429 208734 : TYPE_UNSIGNED (scalar_type));
13430 :
13431 : /* If we felt back to using the mode fail if there was
13432 : no scalar type for it. */
13433 30714855 : if (scalar_type == NULL_TREE)
13434 : return NULL_TREE;
13435 :
13436 : /* If no prevailing mode was supplied, use the mode the target prefers.
13437 : Otherwise lookup a vector mode based on the prevailing mode. */
13438 30714855 : if (prevailing_mode == VOIDmode)
13439 : {
13440 1182371 : gcc_assert (known_eq (nunits, 0U));
13441 1182371 : simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
13442 1182371 : if (SCALAR_INT_MODE_P (simd_mode))
13443 : {
13444 : /* Traditional behavior is not to take the integer mode
13445 : literally, but simply to use it as a way of determining
13446 : the vector size. It is up to mode_for_vector to decide
13447 : what the TYPE_MODE should be.
13448 :
13449 : Note that nunits == 1 is allowed in order to support single
13450 : element vector types. */
13451 65710 : if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
13452 32855 : || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13453 : return NULL_TREE;
13454 : }
13455 : }
13456 29532484 : else if (SCALAR_INT_MODE_P (prevailing_mode)
13457 29532484 : || !related_vector_mode (prevailing_mode,
13458 29531827 : inner_mode, nunits).exists (&simd_mode))
13459 : {
13460 : /* Fall back to using mode_for_vector, mostly in the hope of being
13461 : able to use an integer mode. */
13462 2413570 : if (known_eq (nunits, 0U)
13463 4821536 : && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
13464 : return NULL_TREE;
13465 :
13466 240989 : if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13467 225605 : return NULL_TREE;
13468 : }
13469 :
13470 28284476 : vectype = build_vector_type_for_mode (scalar_type, simd_mode);
13471 :
13472 : /* In cases where the mode was chosen by mode_for_vector, check that
13473 : the target actually supports the chosen mode, or that it at least
13474 : allows the vector mode to be replaced by a like-sized integer. */
13475 56568952 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
13476 28300169 : && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
13477 : return NULL_TREE;
13478 :
13479 : /* Re-attach the address-space qualifier if we canonicalized the scalar
13480 : type. */
13481 28276261 : if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
13482 3 : return build_qualified_type
13483 3 : (vectype, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type)));
13484 :
13485 : return vectype;
13486 : }
13487 :
13488 : /* Function get_vectype_for_scalar_type.
13489 :
13490 : Returns the vector type corresponding to SCALAR_TYPE as supported
13491 : by the target. If GROUP_SIZE is nonzero and we're performing BB
13492 : vectorization, make sure that the number of elements in the vector
13493 : is no bigger than GROUP_SIZE. */
13494 :
13495 : tree
13496 27307675 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
13497 : unsigned int group_size)
13498 : {
13499 : /* For BB vectorization, we should always have a group size once we've
13500 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13501 : are tentative requests during things like early data reference
13502 : analysis and pattern recognition. */
13503 27307675 : if (is_a <bb_vec_info> (vinfo))
13504 24165449 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
13505 : else
13506 : group_size = 0;
13507 :
13508 27307675 : tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13509 : scalar_type);
13510 27307675 : if (vectype && vinfo->vector_mode == VOIDmode)
13511 1098115 : vinfo->vector_mode = TYPE_MODE (vectype);
13512 :
13513 : /* Register the natural choice of vector type, before the group size
13514 : has been applied. */
13515 0 : if (vectype)
13516 24490235 : vinfo->used_vector_modes.add (TYPE_MODE (vectype));
13517 :
13518 : /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13519 : try again with an explicit number of elements. */
13520 24490235 : if (vectype
13521 24490235 : && group_size
13522 27307675 : && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
13523 : {
13524 : /* Start with the biggest number of units that fits within
13525 : GROUP_SIZE and halve it until we find a valid vector type.
13526 : Usually either the first attempt will succeed or all will
13527 : fail (in the latter case because GROUP_SIZE is too small
13528 : for the target), but it's possible that a target could have
13529 : a hole between supported vector types.
13530 :
13531 : If GROUP_SIZE is not a power of 2, this has the effect of
13532 : trying the largest power of 2 that fits within the group,
13533 : even though the group is not a multiple of that vector size.
13534 : The BB vectorizer will then try to carve up the group into
13535 : smaller pieces. */
13536 2592189 : unsigned int nunits = 1 << floor_log2 (group_size);
13537 2592189 : do
13538 : {
13539 2592189 : vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13540 2592189 : scalar_type, nunits);
13541 2592189 : nunits /= 2;
13542 : }
13543 2592189 : while (nunits > 1 && !vectype);
13544 : }
13545 :
13546 27307675 : return vectype;
13547 : }
13548 :
13549 : /* Return the vector type corresponding to SCALAR_TYPE as supported
13550 : by the target. NODE, if nonnull, is the SLP tree node that will
13551 : use the returned vector type. */
13552 :
13553 : tree
13554 183111 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
13555 : {
13556 183111 : unsigned int group_size = 0;
13557 183111 : if (node)
13558 183111 : group_size = SLP_TREE_LANES (node);
13559 183111 : return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13560 : }
13561 :
13562 : /* Function get_mask_type_for_scalar_type.
13563 :
13564 : Returns the mask type corresponding to a result of comparison
13565 : of vectors of specified SCALAR_TYPE as supported by target.
13566 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
13567 : make sure that the number of elements in the vector is no bigger
13568 : than GROUP_SIZE. */
13569 :
13570 : tree
13571 1209549 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13572 : unsigned int group_size)
13573 : {
13574 1209549 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13575 :
13576 1209549 : if (!vectype)
13577 : return NULL;
13578 :
13579 1188657 : return truth_type_for (vectype);
13580 : }
13581 :
13582 : /* Function get_mask_type_for_scalar_type.
13583 :
13584 : Returns the mask type corresponding to a result of comparison
13585 : of vectors of specified SCALAR_TYPE as supported by target.
13586 : NODE, if nonnull, is the SLP tree node that will use the returned
13587 : vector type. */
13588 :
13589 : tree
13590 19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13591 : slp_tree node)
13592 : {
13593 19 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
13594 :
13595 19 : if (!vectype)
13596 : return NULL;
13597 :
13598 19 : return truth_type_for (vectype);
13599 : }
13600 :
13601 : /* Function get_same_sized_vectype
13602 :
13603 : Returns a vector type corresponding to SCALAR_TYPE of size
13604 : VECTOR_TYPE if supported by the target. */
13605 :
13606 : tree
13607 1108 : get_same_sized_vectype (tree scalar_type, tree vector_type)
13608 : {
13609 1108 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
13610 0 : return truth_type_for (vector_type);
13611 :
13612 1108 : poly_uint64 nunits;
13613 2216 : if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
13614 2216 : GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
13615 : return NULL_TREE;
13616 :
13617 1108 : return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
13618 1108 : scalar_type, nunits);
13619 : }
13620 :
13621 : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13622 : would not change the chosen vector modes. */
13623 :
13624 : bool
13625 1676324 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
13626 : {
13627 1676324 : for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
13628 2759102 : i != vinfo->used_vector_modes.end (); ++i)
13629 1980643 : if (!VECTOR_MODE_P (*i)
13630 5941659 : || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
13631 897865 : return false;
13632 778459 : return true;
13633 : }
13634 :
13635 : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
13636 : change the chosen vector modes for analysis of a loop. */
13637 :
13638 : bool
13639 398584 : vect_chooses_same_modes_p (machine_mode vector_mode,
13640 : machine_mode alt_vector_mode)
13641 : {
13642 64502 : return (VECTOR_MODE_P (vector_mode)
13643 398584 : && VECTOR_MODE_P (alt_vector_mode)
13644 797168 : && (related_vector_mode (vector_mode,
13645 : GET_MODE_INNER (alt_vector_mode))
13646 398584 : == alt_vector_mode)
13647 425470 : && (related_vector_mode (alt_vector_mode,
13648 : GET_MODE_INNER (vector_mode))
13649 13443 : == vector_mode));
13650 : }
13651 :
13652 : /* Function vect_is_simple_use.
13653 :
13654 : Input:
13655 : VINFO - the vect info of the loop or basic block that is being vectorized.
13656 : OPERAND - operand in the loop or bb.
13657 : Output:
13658 : DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13659 : case OPERAND is an SSA_NAME that is defined in the vectorizable region
13660 : DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13661 : the definition could be anywhere in the function
13662 : DT - the type of definition
13663 :
13664 : Returns whether a stmt with OPERAND can be vectorized.
13665 : For loops, supportable operands are constants, loop invariants, and operands
13666 : that are defined by the current iteration of the loop. Unsupportable
13667 : operands are those that are defined by a previous iteration of the loop (as
13668 : is the case in reduction/induction computations).
13669 : For basic blocks, supportable operands are constants and bb invariants.
13670 : For now, operands defined outside the basic block are not supported. */
13671 :
13672 : bool
13673 44564520 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
13674 : stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
13675 : {
13676 44564520 : if (def_stmt_info_out)
13677 42514817 : *def_stmt_info_out = NULL;
13678 44564520 : if (def_stmt_out)
13679 10333119 : *def_stmt_out = NULL;
13680 44564520 : *dt = vect_unknown_def_type;
13681 :
13682 44564520 : if (dump_enabled_p ())
13683 : {
13684 805772 : dump_printf_loc (MSG_NOTE, vect_location,
13685 : "vect_is_simple_use: operand ");
13686 805772 : if (TREE_CODE (operand) == SSA_NAME
13687 805772 : && !SSA_NAME_IS_DEFAULT_DEF (operand))
13688 737004 : dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
13689 : else
13690 68768 : dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
13691 : }
13692 :
13693 44564520 : if (CONSTANT_CLASS_P (operand))
13694 3373356 : *dt = vect_constant_def;
13695 41191164 : else if (is_gimple_min_invariant (operand))
13696 400344 : *dt = vect_external_def;
13697 40790820 : else if (TREE_CODE (operand) != SSA_NAME)
13698 1052 : *dt = vect_unknown_def_type;
13699 40789768 : else if (SSA_NAME_IS_DEFAULT_DEF (operand))
13700 723381 : *dt = vect_external_def;
13701 : else
13702 : {
13703 40066387 : gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
13704 40066387 : stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
13705 40066387 : if (!stmt_vinfo)
13706 892866 : *dt = vect_external_def;
13707 : else
13708 : {
13709 39173521 : stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
13710 39173521 : def_stmt = stmt_vinfo->stmt;
13711 39173521 : *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
13712 39173521 : if (def_stmt_info_out)
13713 37132751 : *def_stmt_info_out = stmt_vinfo;
13714 : }
13715 40066387 : if (def_stmt_out)
13716 10105298 : *def_stmt_out = def_stmt;
13717 : }
13718 :
13719 44564520 : if (dump_enabled_p ())
13720 : {
13721 805772 : dump_printf (MSG_NOTE, ", type of def: ");
13722 805772 : switch (*dt)
13723 : {
13724 0 : case vect_uninitialized_def:
13725 0 : dump_printf (MSG_NOTE, "uninitialized\n");
13726 0 : break;
13727 57444 : case vect_constant_def:
13728 57444 : dump_printf (MSG_NOTE, "constant\n");
13729 57444 : break;
13730 27490 : case vect_external_def:
13731 27490 : dump_printf (MSG_NOTE, "external\n");
13732 27490 : break;
13733 575764 : case vect_internal_def:
13734 575764 : dump_printf (MSG_NOTE, "internal\n");
13735 575764 : break;
13736 113076 : case vect_induction_def:
13737 113076 : dump_printf (MSG_NOTE, "induction\n");
13738 113076 : break;
13739 28607 : case vect_reduction_def:
13740 28607 : dump_printf (MSG_NOTE, "reduction\n");
13741 28607 : break;
13742 482 : case vect_double_reduction_def:
13743 482 : dump_printf (MSG_NOTE, "double reduction\n");
13744 482 : break;
13745 2180 : case vect_nested_cycle:
13746 2180 : dump_printf (MSG_NOTE, "nested cycle\n");
13747 2180 : break;
13748 277 : case vect_first_order_recurrence:
13749 277 : dump_printf (MSG_NOTE, "first order recurrence\n");
13750 277 : break;
13751 0 : case vect_condition_def:
13752 0 : dump_printf (MSG_NOTE, "control flow\n");
13753 0 : break;
13754 452 : case vect_unknown_def_type:
13755 452 : dump_printf (MSG_NOTE, "unknown\n");
13756 452 : break;
13757 : }
13758 : }
13759 :
13760 44564520 : if (*dt == vect_unknown_def_type)
13761 : {
13762 56502 : if (dump_enabled_p ())
13763 452 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13764 : "Unsupported pattern.\n");
13765 : return false;
13766 : }
13767 :
13768 : return true;
13769 : }
13770 :
13771 : /* Function vect_is_simple_use.
13772 :
13773 : Same as vect_is_simple_use but determines the operand by operand
13774 : position OPERAND from either STMT or SLP_NODE, filling in *OP
13775 : and *SLP_DEF (when SLP_NODE is not NULL). */
13776 :
13777 : bool
13778 3736514 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13779 : unsigned operand, tree *op, slp_tree *slp_def,
13780 : enum vect_def_type *dt,
13781 : tree *vectype, stmt_vec_info *def_stmt_info_out)
13782 : {
13783 3736514 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
13784 3736514 : *slp_def = child;
13785 3736514 : *vectype = SLP_TREE_VECTYPE (child);
13786 3736514 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
13787 : {
13788 : /* ??? VEC_PERM nodes might be intermediate and their lane value
13789 : have no representative (nor do we build a VEC_PERM stmt for
13790 : the actual operation). Note for two-operator nodes we set
13791 : a representative but leave scalar stmts empty as we'd only
13792 : have one for a subset of lanes. Ideally no caller would
13793 : require *op for internal defs. */
13794 2037346 : if (SLP_TREE_REPRESENTATIVE (child))
13795 : {
13796 1980048 : *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
13797 1980048 : return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
13798 : }
13799 : else
13800 : {
13801 57298 : gcc_assert (SLP_TREE_PERMUTE_P (child));
13802 57298 : *op = error_mark_node;
13803 57298 : *dt = vect_internal_def;
13804 57298 : if (def_stmt_info_out)
13805 152 : *def_stmt_info_out = NULL;
13806 : return true;
13807 : }
13808 : }
13809 : else
13810 : {
13811 1699168 : if (def_stmt_info_out)
13812 52777 : *def_stmt_info_out = NULL;
13813 1699168 : *op = SLP_TREE_SCALAR_OPS (child)[0];
13814 1699168 : *dt = SLP_TREE_DEF_TYPE (child);
13815 1699168 : return true;
13816 : }
13817 : }
13818 :
13819 : /* Wrapper around vect_is_simple_use that elides the scalar op output. */
13820 :
13821 : bool
13822 281951 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13823 : unsigned operand, slp_tree *slp_def,
13824 : enum vect_def_type *dt, tree *vectype)
13825 : {
13826 281951 : tree op;
13827 281951 : return vect_is_simple_use (vinfo, slp_node, operand, &op, slp_def, dt,
13828 281951 : vectype);
13829 : }
13830 :
13831 : /* If OP is not NULL and is external or constant update its vector
13832 : type with VECTYPE. Returns true if successful or false if not,
13833 : for example when conflicting vector types are present. */
13834 :
13835 : bool
13836 3253541 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
13837 : {
13838 3253541 : if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
13839 : return true;
13840 1119142 : if (SLP_TREE_VECTYPE (op))
13841 23430 : return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
13842 : /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13843 : should be handled by patters. Allow vect_constant_def for now
13844 : as well as the trivial single-lane uniform vect_external_def case
13845 : both of which we code-generate reasonably. */
13846 1095712 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
13847 1721 : && SLP_TREE_DEF_TYPE (op) == vect_external_def
13848 1096966 : && SLP_TREE_LANES (op) > 1)
13849 : return false;
13850 1095514 : SLP_TREE_VECTYPE (op) = vectype;
13851 1095514 : return true;
13852 : }
13853 :
13854 : /* Function supportable_widening_operation
13855 :
13856 : Check whether an operation represented by the code CODE is a
13857 : widening operation that is supported by the target platform in
13858 : vector form (i.e., when operating on arguments of type VECTYPE_IN
13859 : producing a result of type VECTYPE_OUT).
13860 :
13861 : Widening operations we currently support are NOP (CONVERT), FLOAT,
13862 : FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13863 : are supported by the target platform either directly (via vector
13864 : tree-codes), or via target builtins.
13865 :
13866 : When EVENODD_OK then also lane-swizzling operations are considered.
13867 :
13868 : Output:
13869 : - CODE1 and CODE2 are codes of vector operations to be used when
13870 : vectorizing the operation, if available.
13871 : - MULTI_STEP_CVT determines the number of required intermediate steps in
13872 : case of multi-step conversion (like char->short->int - in that case
13873 : MULTI_STEP_CVT will be 1).
13874 : - INTERM_TYPES contains the intermediate type required to perform the
13875 : widening operation (short in the above example). */
13876 :
13877 : bool
13878 551978 : supportable_widening_operation (code_helper code,
13879 : tree vectype_out, tree vectype_in,
13880 : bool evenodd_ok,
13881 : code_helper *code1,
13882 : code_helper *code2,
13883 : int *multi_step_cvt,
13884 : vec<tree> *interm_types)
13885 : {
13886 551978 : machine_mode vec_mode;
13887 551978 : enum insn_code icode1, icode2;
13888 551978 : optab optab1 = unknown_optab, optab2 = unknown_optab;
13889 551978 : tree vectype = vectype_in;
13890 551978 : tree wide_vectype = vectype_out;
13891 551978 : tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
13892 551978 : int i;
13893 551978 : tree prev_type, intermediate_type;
13894 551978 : machine_mode intermediate_mode, prev_mode;
13895 551978 : optab optab3, optab4;
13896 :
13897 551978 : *multi_step_cvt = 0;
13898 :
13899 551978 : switch (code.safe_as_tree_code ())
13900 : {
13901 : case MAX_TREE_CODES:
13902 : /* Don't set c1 and c2 if code is not a tree_code. */
13903 : break;
13904 :
13905 209503 : case WIDEN_MULT_EXPR:
13906 : /* The result of a vectorized widening operation usually requires
13907 : two vectors (because the widened results do not fit into one vector).
13908 : The generated vector results would normally be expected to be
13909 : generated in the same order as in the original scalar computation,
13910 : i.e. if 8 results are generated in each vector iteration, they are
13911 : to be organized as follows:
13912 : vect1: [res1,res2,res3,res4],
13913 : vect2: [res5,res6,res7,res8].
13914 :
13915 : However, in the special case that the result of the widening
13916 : operation is used in a reduction computation only, the order doesn't
13917 : matter (because when vectorizing a reduction we change the order of
13918 : the computation). Some targets can take advantage of this and
13919 : generate more efficient code. For example, targets like Altivec,
13920 : that support widen_mult using a sequence of {mult_even,mult_odd}
13921 : generate the following vectors:
13922 : vect1: [res1,res3,res5,res7],
13923 : vect2: [res2,res4,res6,res8].
13924 :
13925 : When vectorizing outer-loops, we execute the inner-loop sequentially
13926 : (each vectorized inner-loop iteration contributes to VF outer-loop
13927 : iterations in parallel). We therefore don't allow to change the
13928 : order of the computation in the inner-loop during outer-loop
13929 : vectorization. */
13930 : /* TODO: Another case in which order doesn't *really* matter is when we
13931 : widen and then contract again, e.g. (short)((int)x * y >> 8).
13932 : Normally, pack_trunc performs an even/odd permute, whereas the
13933 : repack from an even/odd expansion would be an interleave, which
13934 : would be significantly simpler for e.g. AVX2. */
13935 : /* In any case, in order to avoid duplicating the code below, recurse
13936 : on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13937 : are properly set up for the caller. If we fail, we'll continue with
13938 : a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13939 209503 : if (evenodd_ok
13940 209503 : && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
13941 : vectype_out, vectype_in,
13942 : evenodd_ok, code1,
13943 : code2, multi_step_cvt,
13944 : interm_types))
13945 107367 : return true;
13946 : c1 = VEC_WIDEN_MULT_LO_EXPR;
13947 : c2 = VEC_WIDEN_MULT_HI_EXPR;
13948 : break;
13949 :
13950 : case DOT_PROD_EXPR:
13951 444611 : c1 = DOT_PROD_EXPR;
13952 444611 : c2 = DOT_PROD_EXPR;
13953 : break;
13954 :
13955 0 : case SAD_EXPR:
13956 0 : c1 = SAD_EXPR;
13957 0 : c2 = SAD_EXPR;
13958 0 : break;
13959 :
13960 207397 : case VEC_WIDEN_MULT_EVEN_EXPR:
13961 : /* Support the recursion induced just above. */
13962 207397 : c1 = VEC_WIDEN_MULT_EVEN_EXPR;
13963 207397 : c2 = VEC_WIDEN_MULT_ODD_EXPR;
13964 207397 : break;
13965 :
13966 9852 : case WIDEN_LSHIFT_EXPR:
13967 9852 : c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
13968 9852 : c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
13969 9852 : break;
13970 :
13971 45205 : CASE_CONVERT:
13972 45205 : c1 = VEC_UNPACK_LO_EXPR;
13973 45205 : c2 = VEC_UNPACK_HI_EXPR;
13974 45205 : break;
13975 :
13976 9267 : case FLOAT_EXPR:
13977 9267 : c1 = VEC_UNPACK_FLOAT_LO_EXPR;
13978 9267 : c2 = VEC_UNPACK_FLOAT_HI_EXPR;
13979 9267 : break;
13980 :
13981 127 : case FIX_TRUNC_EXPR:
13982 127 : c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
13983 127 : c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
13984 127 : break;
13985 :
13986 0 : default:
13987 0 : gcc_unreachable ();
13988 : }
13989 :
13990 444611 : if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
13991 : std::swap (c1, c2);
13992 :
13993 444611 : if (code == FIX_TRUNC_EXPR)
13994 : {
13995 : /* The signedness is determined from output operand. */
13996 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
13997 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
13998 : }
13999 781474 : else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
14000 45205 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14001 7954 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14002 7954 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14003 374353 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14004 : {
14005 : /* If the input and result modes are the same, a different optab
14006 : is needed where we pass in the number of units in vectype. */
14007 : optab1 = vec_unpacks_sbool_lo_optab;
14008 : optab2 = vec_unpacks_sbool_hi_optab;
14009 : }
14010 :
14011 444611 : vec_mode = TYPE_MODE (vectype);
14012 444611 : if (widening_fn_p (code))
14013 : {
14014 : /* If this is an internal fn then we must check whether the target
14015 : supports either a low-high split or an even-odd split. */
14016 70627 : internal_fn ifn = as_internal_fn ((combined_fn) code);
14017 :
14018 70627 : internal_fn lo, hi, even, odd;
14019 70627 : lookup_hilo_internal_fn (ifn, &lo, &hi);
14020 70627 : if (BYTES_BIG_ENDIAN)
14021 : std::swap (lo, hi);
14022 70627 : *code1 = as_combined_fn (lo);
14023 70627 : *code2 = as_combined_fn (hi);
14024 70627 : optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
14025 70627 : optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
14026 :
14027 : /* If we don't support low-high, then check for even-odd. */
14028 70627 : if (!optab1
14029 70627 : || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14030 0 : || !optab2
14031 70627 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14032 : {
14033 70627 : lookup_evenodd_internal_fn (ifn, &even, &odd);
14034 70627 : *code1 = as_combined_fn (even);
14035 70627 : *code2 = as_combined_fn (odd);
14036 70627 : optab1 = direct_internal_fn_optab (even, {vectype, vectype});
14037 70627 : optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
14038 : }
14039 : }
14040 373984 : else if (code.is_tree_code ())
14041 : {
14042 373984 : if (code == FIX_TRUNC_EXPR)
14043 : {
14044 : /* The signedness is determined from output operand. */
14045 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14046 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14047 : }
14048 373857 : else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
14049 45205 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14050 7954 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14051 7954 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14052 374353 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14053 : {
14054 : /* If the input and result modes are the same, a different optab
14055 : is needed where we pass in the number of units in vectype. */
14056 : optab1 = vec_unpacks_sbool_lo_optab;
14057 : optab2 = vec_unpacks_sbool_hi_optab;
14058 : }
14059 : else
14060 : {
14061 373361 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14062 373361 : optab2 = optab_for_tree_code (c2, vectype, optab_default);
14063 : }
14064 373984 : *code1 = c1;
14065 373984 : *code2 = c2;
14066 : }
14067 :
14068 444611 : if (!optab1 || !optab2)
14069 : return false;
14070 :
14071 444611 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14072 444611 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14073 : return false;
14074 :
14075 :
14076 167230 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14077 167230 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14078 : {
14079 155483 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14080 : return true;
14081 : /* For scalar masks we may have different boolean
14082 : vector types having the same QImode. Thus we
14083 : add additional check for elements number. */
14084 4156 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
14085 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14086 : return true;
14087 : }
14088 :
14089 : /* Check if it's a multi-step conversion that can be done using intermediate
14090 : types. */
14091 :
14092 11976 : prev_type = vectype;
14093 11976 : prev_mode = vec_mode;
14094 :
14095 11976 : if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
14096 : return false;
14097 :
14098 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14099 : intermediate steps in promotion sequence. We try
14100 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14101 : not. */
14102 11924 : interm_types->create (MAX_INTERM_CVT_STEPS);
14103 25170 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14104 : {
14105 13246 : intermediate_mode = insn_data[icode1].operand[0].mode;
14106 13246 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14107 4695 : intermediate_type
14108 4695 : = vect_halve_mask_nunits (prev_type, intermediate_mode);
14109 8551 : else if (VECTOR_MODE_P (intermediate_mode))
14110 : {
14111 8551 : tree intermediate_element_type
14112 8551 : = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
14113 8551 : TYPE_UNSIGNED (prev_type));
14114 8551 : intermediate_type
14115 8551 : = build_vector_type_for_mode (intermediate_element_type,
14116 : intermediate_mode);
14117 8551 : }
14118 : else
14119 0 : intermediate_type
14120 0 : = lang_hooks.types.type_for_mode (intermediate_mode,
14121 0 : TYPE_UNSIGNED (prev_type));
14122 :
14123 13246 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14124 4695 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14125 4695 : && intermediate_mode == TYPE_MODE (wide_vectype)
14126 13557 : && SCALAR_INT_MODE_P (intermediate_mode))
14127 : {
14128 : /* If the input and result modes are the same, a different optab
14129 : is needed where we pass in the number of units in vectype. */
14130 : optab3 = vec_unpacks_sbool_lo_optab;
14131 : optab4 = vec_unpacks_sbool_hi_optab;
14132 : }
14133 : else
14134 : {
14135 12935 : optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
14136 12935 : optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
14137 : }
14138 :
14139 13246 : if (!optab3 || !optab4
14140 13246 : || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
14141 13206 : || insn_data[icode1].operand[0].mode != intermediate_mode
14142 13206 : || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
14143 13206 : || insn_data[icode2].operand[0].mode != intermediate_mode
14144 13206 : || ((icode1 = optab_handler (optab3, intermediate_mode))
14145 : == CODE_FOR_nothing)
14146 26064 : || ((icode2 = optab_handler (optab4, intermediate_mode))
14147 : == CODE_FOR_nothing))
14148 : break;
14149 :
14150 12818 : interm_types->quick_push (intermediate_type);
14151 12818 : (*multi_step_cvt)++;
14152 :
14153 12818 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14154 12818 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14155 : {
14156 11578 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14157 : return true;
14158 3770 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
14159 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14160 : return true;
14161 : }
14162 :
14163 1322 : prev_type = intermediate_type;
14164 1322 : prev_mode = intermediate_mode;
14165 : }
14166 :
14167 428 : interm_types->release ();
14168 428 : return false;
14169 : }
14170 :
14171 :
14172 : /* Function supportable_narrowing_operation
14173 :
14174 : Check whether an operation represented by the code CODE is a
14175 : narrowing operation that is supported by the target platform in
14176 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14177 : and producing a result of type VECTYPE_OUT).
14178 :
14179 : Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14180 : and FLOAT. This function checks if these operations are supported by
14181 : the target platform directly via vector tree-codes.
14182 :
14183 : Output:
14184 : - CODE1 is the code of a vector operation to be used when
14185 : vectorizing the operation, if available.
14186 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14187 : case of multi-step conversion (like int->short->char - in that case
14188 : MULTI_STEP_CVT will be 1).
14189 : - INTERM_TYPES contains the intermediate type required to perform the
14190 : narrowing operation (short in the above example). */
14191 :
14192 : bool
14193 45046 : supportable_narrowing_operation (code_helper code,
14194 : tree vectype_out, tree vectype_in,
14195 : code_helper *code1, int *multi_step_cvt,
14196 : vec<tree> *interm_types)
14197 : {
14198 45046 : machine_mode vec_mode;
14199 45046 : enum insn_code icode1;
14200 45046 : optab optab1, interm_optab;
14201 45046 : tree vectype = vectype_in;
14202 45046 : tree narrow_vectype = vectype_out;
14203 45046 : enum tree_code c1;
14204 45046 : tree intermediate_type, prev_type;
14205 45046 : machine_mode intermediate_mode, prev_mode;
14206 45046 : int i;
14207 45046 : unsigned HOST_WIDE_INT n_elts;
14208 45046 : bool uns;
14209 :
14210 45046 : if (!code.is_tree_code ())
14211 : return false;
14212 :
14213 45046 : *multi_step_cvt = 0;
14214 45046 : switch ((tree_code) code)
14215 : {
14216 43536 : CASE_CONVERT:
14217 43536 : c1 = VEC_PACK_TRUNC_EXPR;
14218 43536 : if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
14219 11640 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14220 11640 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
14221 5255 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
14222 48791 : && n_elts < BITS_PER_UNIT)
14223 : optab1 = vec_pack_sbool_trunc_optab;
14224 : else
14225 41053 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14226 : break;
14227 :
14228 572 : case FIX_TRUNC_EXPR:
14229 572 : c1 = VEC_PACK_FIX_TRUNC_EXPR;
14230 : /* The signedness is determined from output operand. */
14231 572 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14232 572 : break;
14233 :
14234 938 : case FLOAT_EXPR:
14235 938 : c1 = VEC_PACK_FLOAT_EXPR;
14236 938 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14237 938 : break;
14238 :
14239 0 : default:
14240 0 : gcc_unreachable ();
14241 : }
14242 :
14243 45046 : if (!optab1)
14244 : return false;
14245 :
14246 45046 : vec_mode = TYPE_MODE (vectype);
14247 45046 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
14248 : return false;
14249 :
14250 39565 : *code1 = c1;
14251 :
14252 39565 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14253 : {
14254 24968 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14255 : return true;
14256 : /* For scalar masks we may have different boolean
14257 : vector types having the same QImode. Thus we
14258 : add additional check for elements number. */
14259 5847 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
14260 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14261 : return true;
14262 : }
14263 :
14264 14748 : if (code == FLOAT_EXPR)
14265 : return false;
14266 :
14267 : /* Check if it's a multi-step conversion that can be done using intermediate
14268 : types. */
14269 14748 : prev_mode = vec_mode;
14270 14748 : prev_type = vectype;
14271 14748 : if (code == FIX_TRUNC_EXPR)
14272 94 : uns = TYPE_UNSIGNED (vectype_out);
14273 : else
14274 14654 : uns = TYPE_UNSIGNED (vectype);
14275 :
14276 : /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14277 : conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14278 : costly than signed. */
14279 14748 : if (code == FIX_TRUNC_EXPR && uns)
14280 : {
14281 28 : enum insn_code icode2;
14282 :
14283 28 : intermediate_type
14284 28 : = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
14285 28 : interm_optab
14286 28 : = optab_for_tree_code (c1, intermediate_type, optab_default);
14287 28 : if (interm_optab != unknown_optab
14288 28 : && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
14289 28 : && insn_data[icode1].operand[0].mode
14290 28 : == insn_data[icode2].operand[0].mode)
14291 : {
14292 : uns = false;
14293 : optab1 = interm_optab;
14294 : icode1 = icode2;
14295 : }
14296 : }
14297 :
14298 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14299 : intermediate steps in promotion sequence. We try
14300 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14301 14748 : interm_types->create (MAX_INTERM_CVT_STEPS);
14302 31580 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14303 : {
14304 16832 : intermediate_mode = insn_data[icode1].operand[0].mode;
14305 16832 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14306 7024 : intermediate_type
14307 7024 : = vect_double_mask_nunits (prev_type, intermediate_mode);
14308 : else
14309 9808 : intermediate_type
14310 9808 : = lang_hooks.types.type_for_mode (intermediate_mode, uns);
14311 16832 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14312 7024 : && VECTOR_BOOLEAN_TYPE_P (prev_type)
14313 7024 : && SCALAR_INT_MODE_P (prev_mode)
14314 3128 : && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
14315 19960 : && n_elts < BITS_PER_UNIT)
14316 : interm_optab = vec_pack_sbool_trunc_optab;
14317 : else
14318 16480 : interm_optab
14319 16480 : = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
14320 : optab_default);
14321 352 : if (!interm_optab
14322 16832 : || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
14323 16832 : || insn_data[icode1].operand[0].mode != intermediate_mode
14324 33312 : || ((icode1 = optab_handler (interm_optab, intermediate_mode))
14325 : == CODE_FOR_nothing))
14326 : break;
14327 :
14328 15917 : interm_types->quick_push (intermediate_type);
14329 15917 : (*multi_step_cvt)++;
14330 :
14331 15917 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14332 : {
14333 13833 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14334 : return true;
14335 4958 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
14336 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14337 : return true;
14338 : }
14339 :
14340 2084 : prev_mode = intermediate_mode;
14341 2084 : prev_type = intermediate_type;
14342 2084 : optab1 = interm_optab;
14343 : }
14344 :
14345 915 : interm_types->release ();
14346 915 : return false;
14347 : }
14348 :
14349 : /* Function supportable_indirect_convert_operation
14350 :
14351 : Check whether an operation represented by the code CODE is single or multi
14352 : operations that are supported by the target platform in
14353 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14354 : producing a result of type VECTYPE_OUT).
14355 :
14356 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14357 : This function checks if these operations are supported
14358 : by the target platform directly (via vector tree-codes).
14359 :
14360 : Output:
14361 : - converts contains some pairs to perform the convert operation,
14362 : the pair's first is the intermediate type, and its second is the code of
14363 : a vector operation to be used when converting the operation from the
14364 : previous type to the intermediate type. */
14365 : bool
14366 94203 : supportable_indirect_convert_operation (code_helper code,
14367 : tree vectype_out,
14368 : tree vectype_in,
14369 : vec<std::pair<tree, tree_code> > &converts,
14370 : slp_tree slp_op0)
14371 : {
14372 94203 : bool found_mode = false;
14373 94203 : scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
14374 94203 : scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
14375 94203 : tree_code code1, code2;
14376 :
14377 94203 : tree cvt_type = NULL_TREE;
14378 94203 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
14379 :
14380 94203 : if (supportable_convert_operation ((tree_code) code,
14381 : vectype_out,
14382 : vectype_in))
14383 : {
14384 86638 : converts.safe_push (std::make_pair (vectype_out, (tree_code) code));
14385 86638 : return true;
14386 : }
14387 :
14388 : /* For conversions between mask types where the destination has
14389 : a data mode attempt a vcond_mask conversion. */
14390 7565 : if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
14391 145 : && VECTOR_BOOLEAN_TYPE_P (vectype_out)
14392 7710 : && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
14393 : {
14394 144 : tree scalar_datatype
14395 144 : = build_nonstandard_integer_type (element_precision (vectype_out), 0);
14396 144 : tree datatype_out = build_vector_type_for_mode (scalar_datatype,
14397 144 : TYPE_MODE (vectype_out));
14398 144 : if (expand_vec_cond_expr_p (datatype_out, vectype_in))
14399 : {
14400 2 : converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
14401 2 : return true;
14402 : }
14403 : }
14404 :
14405 : /* For conversions between float and integer types try whether
14406 : we can use intermediate signed integer types to support the
14407 : conversion. */
14408 15126 : if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
14409 7563 : && (code == FLOAT_EXPR
14410 4924 : || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
14411 : {
14412 1592 : bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
14413 796 : bool float_expr_p = code == FLOAT_EXPR;
14414 796 : unsigned short target_size;
14415 796 : scalar_mode intermediate_mode;
14416 796 : if (demotion)
14417 : {
14418 99 : intermediate_mode = lhs_mode;
14419 99 : target_size = GET_MODE_SIZE (rhs_mode);
14420 : }
14421 : else
14422 : {
14423 697 : target_size = GET_MODE_SIZE (lhs_mode);
14424 697 : if (!int_mode_for_size
14425 697 : (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
14426 142 : return false;
14427 : }
14428 796 : code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
14429 729 : code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
14430 796 : opt_scalar_mode mode_iter;
14431 2483 : FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
14432 : {
14433 2483 : intermediate_mode = mode_iter.require ();
14434 :
14435 4966 : if (GET_MODE_SIZE (intermediate_mode) > target_size)
14436 : break;
14437 :
14438 1882 : scalar_mode cvt_mode;
14439 1882 : if (!int_mode_for_size
14440 1882 : (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
14441 : break;
14442 :
14443 1852 : cvt_type = build_nonstandard_integer_type
14444 1852 : (GET_MODE_BITSIZE (cvt_mode), 0);
14445 :
14446 : /* Check if the intermediate type can hold OP0's range.
14447 : When converting from float to integer this is not necessary
14448 : because values that do not fit the (smaller) target type are
14449 : unspecified anyway. */
14450 1852 : if (demotion && float_expr_p)
14451 : {
14452 23 : wide_int op_min_value, op_max_value;
14453 : /* For vector form, it looks like slp_op0 doesn't have RANGE_INFO.
14454 : In the future, if it is supported, changes may need to be made
14455 : to this part, such as checking the RANGE of each element
14456 : in the vector. */
14457 23 : if (slp_op0)
14458 : {
14459 13 : tree def;
14460 : /* ??? Merge ranges in case of more than one lane. */
14461 13 : if (SLP_TREE_LANES (slp_op0) != 1
14462 0 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
14463 13 : || !vect_get_range_info (def,
14464 : &op_min_value, &op_max_value))
14465 : break;
14466 : }
14467 : else
14468 : break;
14469 :
14470 0 : if (cvt_type == NULL_TREE
14471 0 : || (wi::min_precision (op_max_value, SIGNED)
14472 0 : > TYPE_PRECISION (cvt_type))
14473 0 : || (wi::min_precision (op_min_value, SIGNED)
14474 0 : > TYPE_PRECISION (cvt_type)))
14475 0 : continue;
14476 23 : }
14477 :
14478 1829 : cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
14479 : cvt_type,
14480 : nelts);
14481 : /* This should only happened for SLP as long as loop vectorizer
14482 : only supports same-sized vector. */
14483 3516 : if (cvt_type == NULL_TREE
14484 3658 : || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
14485 1829 : || !supportable_convert_operation ((tree_code) code1,
14486 : vectype_out, cvt_type)
14487 2564 : || !supportable_convert_operation ((tree_code) code2,
14488 : cvt_type, vectype_in))
14489 1687 : continue;
14490 :
14491 : found_mode = true;
14492 : break;
14493 : }
14494 :
14495 796 : if (found_mode)
14496 : {
14497 142 : converts.safe_push (std::make_pair (cvt_type, code2));
14498 142 : if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
14499 142 : converts.safe_push (std::make_pair (vectype_out, code1));
14500 : return true;
14501 : }
14502 : }
14503 : return false;
14504 : }
14505 :
14506 : /* Generate and return a vector mask of MASK_TYPE such that
14507 : mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14508 : Add the statements to SEQ. */
14509 :
14510 : tree
14511 0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
14512 : tree end_index, const char *name)
14513 : {
14514 0 : tree cmp_type = TREE_TYPE (start_index);
14515 0 : gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
14516 : cmp_type, mask_type,
14517 : OPTIMIZE_FOR_SPEED));
14518 0 : gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
14519 : start_index, end_index,
14520 : build_zero_cst (mask_type));
14521 0 : tree tmp;
14522 0 : if (name)
14523 0 : tmp = make_temp_ssa_name (mask_type, NULL, name);
14524 : else
14525 0 : tmp = make_ssa_name (mask_type);
14526 0 : gimple_call_set_lhs (call, tmp);
14527 0 : gimple_seq_add_stmt (seq, call);
14528 0 : return tmp;
14529 : }
14530 :
14531 : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14532 : J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14533 :
14534 : tree
14535 0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
14536 : tree end_index)
14537 : {
14538 0 : tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
14539 0 : return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
14540 : }
14541 :
14542 : /* Try to compute the vector types required to vectorize STMT_INFO,
14543 : returning true on success and false if vectorization isn't possible.
14544 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
14545 : take sure that the number of elements in the vectors is no bigger
14546 : than GROUP_SIZE.
14547 :
14548 : On success:
14549 :
14550 : - Set *STMT_VECTYPE_OUT to:
14551 : - NULL_TREE if the statement doesn't need to be vectorized;
14552 : - the equivalent of STMT_VINFO_VECTYPE otherwise. */
14553 :
14554 : opt_result
14555 6191176 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
14556 : tree *stmt_vectype_out,
14557 : unsigned int group_size)
14558 : {
14559 6191176 : gimple *stmt = stmt_info->stmt;
14560 :
14561 : /* For BB vectorization, we should always have a group size once we've
14562 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14563 : are tentative requests during things like early data reference
14564 : analysis and pattern recognition. */
14565 6191176 : if (is_a <bb_vec_info> (vinfo))
14566 4967601 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
14567 : else
14568 : group_size = 0;
14569 :
14570 6191176 : *stmt_vectype_out = NULL_TREE;
14571 :
14572 6191176 : if (gimple_get_lhs (stmt) == NULL_TREE
14573 : /* Allow vector conditionals through here. */
14574 2610 : && !is_a <gcond *> (stmt)
14575 : /* MASK_STORE and friends have no lhs, but are ok. */
14576 6196376 : && !(is_gimple_call (stmt)
14577 2610 : && gimple_call_internal_p (stmt)
14578 2590 : && internal_store_fn_p (gimple_call_internal_fn (stmt))))
14579 : {
14580 20 : if (is_a <gcall *> (stmt))
14581 : {
14582 : /* Ignore calls with no lhs. These must be calls to
14583 : #pragma omp simd functions, and what vectorization factor
14584 : it really needs can't be determined until
14585 : vectorizable_simd_clone_call. */
14586 20 : if (dump_enabled_p ())
14587 18 : dump_printf_loc (MSG_NOTE, vect_location,
14588 : "defer to SIMD clone analysis.\n");
14589 20 : return opt_result::success ();
14590 : }
14591 :
14592 0 : return opt_result::failure_at (stmt,
14593 : "not vectorized: irregular stmt: %G", stmt);
14594 : }
14595 :
14596 6191156 : tree vectype;
14597 6191156 : tree scalar_type = NULL_TREE;
14598 6191156 : if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
14599 : {
14600 1664863 : vectype = STMT_VINFO_VECTYPE (stmt_info);
14601 1664863 : if (dump_enabled_p ())
14602 83992 : dump_printf_loc (MSG_NOTE, vect_location,
14603 : "precomputed vectype: %T\n", vectype);
14604 : }
14605 4526293 : else if (vect_use_mask_type_p (stmt_info))
14606 : {
14607 230032 : unsigned int precision = stmt_info->mask_precision;
14608 230032 : scalar_type = build_nonstandard_integer_type (precision, 1);
14609 230032 : vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
14610 230032 : if (!vectype)
14611 0 : return opt_result::failure_at (stmt, "not vectorized: unsupported"
14612 : " data-type %T\n", scalar_type);
14613 230032 : if (dump_enabled_p ())
14614 5024 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14615 : }
14616 : else
14617 : {
14618 : /* If we got here with a gcond it means that the target had no available vector
14619 : mode for the scalar type. We can't vectorize so abort. */
14620 4296261 : if (is_a <gcond *> (stmt))
14621 0 : return opt_result::failure_at (stmt,
14622 : "not vectorized:"
14623 : " unsupported data-type for gcond %T\n",
14624 : scalar_type);
14625 :
14626 4296261 : if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
14627 1533507 : scalar_type = TREE_TYPE (DR_REF (dr));
14628 : else
14629 2762754 : scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
14630 :
14631 4296261 : if (dump_enabled_p ())
14632 : {
14633 67976 : if (group_size)
14634 9202 : dump_printf_loc (MSG_NOTE, vect_location,
14635 : "get vectype for scalar type (group size %d):"
14636 : " %T\n", group_size, scalar_type);
14637 : else
14638 58774 : dump_printf_loc (MSG_NOTE, vect_location,
14639 : "get vectype for scalar type: %T\n", scalar_type);
14640 : }
14641 4296261 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
14642 4296261 : if (!vectype)
14643 254985 : return opt_result::failure_at (stmt,
14644 : "not vectorized:"
14645 : " unsupported data-type %T\n",
14646 : scalar_type);
14647 :
14648 4041276 : if (dump_enabled_p ())
14649 67717 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14650 : }
14651 :
14652 4355300 : if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
14653 0 : return opt_result::failure_at (stmt,
14654 : "not vectorized: vector stmt in loop:%G",
14655 : stmt);
14656 :
14657 5936171 : *stmt_vectype_out = vectype;
14658 5936171 : return opt_result::success ();
14659 : }
14660 :
14661 : /* Generate and return statement sequence that sets vector length LEN that is:
14662 :
14663 : min_of_start_and_end = min (START_INDEX, END_INDEX);
14664 : left_len = END_INDEX - min_of_start_and_end;
14665 : rhs = min (left_len, LEN_LIMIT);
14666 : LEN = rhs;
14667 :
14668 : Note: the cost of the code generated by this function is modeled
14669 : by vect_estimate_min_profitable_iters, so changes here may need
14670 : corresponding changes there. */
14671 :
14672 : gimple_seq
14673 0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
14674 : {
14675 0 : gimple_seq stmts = NULL;
14676 0 : tree len_type = TREE_TYPE (len);
14677 0 : gcc_assert (TREE_TYPE (start_index) == len_type);
14678 :
14679 0 : tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
14680 0 : tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
14681 0 : tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
14682 0 : gimple* stmt = gimple_build_assign (len, rhs);
14683 0 : gimple_seq_add_stmt (&stmts, stmt);
14684 :
14685 0 : return stmts;
14686 : }
14687 :
|
