Line data Source code
1 : /* Statement Analysis and Transformation for Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "ssa.h"
31 : #include "optabs-tree.h"
32 : #include "insn-config.h"
33 : #include "recog.h" /* FIXME: for insn_data */
34 : #include "cgraph.h"
35 : #include "dumpfile.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "tree-eh.h"
40 : #include "gimplify.h"
41 : #include "gimple-iterator.h"
42 : #include "gimplify-me.h"
43 : #include "tree-cfg.h"
44 : #include "tree-ssa-loop-manip.h"
45 : #include "cfgloop.h"
46 : #include "explow.h"
47 : #include "tree-ssa-loop.h"
48 : #include "tree-scalar-evolution.h"
49 : #include "tree-vectorizer.h"
50 : #include "builtins.h"
51 : #include "internal-fn.h"
52 : #include "tree-vector-builder.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "tree-ssa-loop-niter.h"
56 : #include "gimple-fold.h"
57 : #include "regs.h"
58 : #include "attribs.h"
59 : #include "optabs-libfuncs.h"
60 : #include "tree-dfa.h"
61 : #include "hierarchical_discriminator.h"
62 :
63 : /* For lang_hooks.types.type_for_mode. */
64 : #include "langhooks.h"
65 :
66 : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
67 : bool = false);
68 :
69 : /* Return TRUE iff the given statement is in an inner loop relative to
70 : the loop being vectorized. */
71 : bool
72 6669288 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
73 : {
74 6669288 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
75 6669288 : basic_block bb = gimple_bb (stmt);
76 6669288 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
77 3157574 : class loop* loop;
78 :
79 3157574 : if (!loop_vinfo)
80 : return false;
81 :
82 3157574 : loop = LOOP_VINFO_LOOP (loop_vinfo);
83 :
84 3157574 : return (bb->loop_father == loop->inner);
85 : }
86 :
87 : /* Record the cost of a statement, either by directly informing the
88 : target model or by saving it in a vector for later processing.
89 : Return a preliminary estimate of the statement's cost. */
90 :
91 : unsigned
92 9314254 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
93 : enum vect_cost_for_stmt kind,
94 : stmt_vec_info stmt_info, slp_tree node,
95 : tree vectype, int misalign,
96 : enum vect_cost_model_location where)
97 : {
98 9314254 : if ((kind == vector_load || kind == unaligned_load)
99 1679824 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
100 : kind = vector_gather_load;
101 9314254 : if ((kind == vector_store || kind == unaligned_store)
102 1058961 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
103 9314254 : kind = vector_scatter_store;
104 :
105 9314254 : stmt_info_for_cost si
106 9314254 : = { count, kind, where, stmt_info, node, vectype, misalign };
107 9314254 : body_cost_vec->safe_push (si);
108 :
109 9314254 : return (unsigned)
110 9314254 : (builtin_vectorization_cost (kind, vectype, misalign) * count);
111 : }
112 :
113 : unsigned
114 4331093 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
115 : enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
116 : tree vectype, int misalign,
117 : enum vect_cost_model_location where)
118 : {
119 4331093 : return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
120 4331093 : vectype, misalign, where);
121 : }
122 :
123 : unsigned
124 1803192 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
125 : enum vect_cost_for_stmt kind, slp_tree node,
126 : tree vectype, int misalign,
127 : enum vect_cost_model_location where)
128 : {
129 1803192 : return record_stmt_cost (body_cost_vec, count, kind,
130 : SLP_TREE_REPRESENTATIVE (node), node,
131 1803192 : vectype, misalign, where);
132 : }
133 :
134 : unsigned
135 0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
136 : enum vect_cost_for_stmt kind,
137 : enum vect_cost_model_location where)
138 : {
139 0 : gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
140 : || kind == scalar_stmt);
141 0 : return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
142 0 : NULL_TREE, 0, where);
143 : }
144 :
145 : /* Return a variable of type ELEM_TYPE[NELEMS]. */
146 :
147 : static tree
148 0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
149 : {
150 0 : return create_tmp_var (build_array_type_nelts (elem_type, nelems),
151 0 : "vect_array");
152 : }
153 :
154 : /* ARRAY is an array of vectors created by create_vector_array.
155 : Return an SSA_NAME for the vector in index N. The reference
156 : is part of the vectorization of STMT_INFO and the vector is associated
157 : with scalar destination SCALAR_DEST.
158 : If we need to ensure that inactive elements are set to zero,
159 : NEED_ZEROING is true, MASK contains the loop mask to be used. */
160 :
161 : static tree
162 0 : read_vector_array (vec_info *vinfo,
163 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
164 : tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
165 : bool need_zeroing, tree mask)
166 : {
167 0 : tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
168 0 : gimple *new_stmt;
169 :
170 0 : gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
171 0 : vect_type = TREE_TYPE (TREE_TYPE (array));
172 0 : tmp = vect_create_destination_var (scalar_dest, vect_type);
173 0 : vect = vect_create_destination_var (scalar_dest, vect_type);
174 0 : array_ref = build4 (ARRAY_REF, vect_type, array,
175 0 : build_int_cst (size_type_node, n),
176 : NULL_TREE, NULL_TREE);
177 :
178 0 : new_stmt = gimple_build_assign (tmp, array_ref);
179 0 : tmp_name = make_ssa_name (vect, new_stmt);
180 0 : gimple_assign_set_lhs (new_stmt, tmp_name);
181 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
182 :
183 0 : if (need_zeroing)
184 : {
185 0 : tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
186 : vect_type);
187 0 : vect_name = make_ssa_name (vect, new_stmt);
188 0 : new_stmt
189 0 : = gimple_build_assign (vect_name, VEC_COND_EXPR,
190 : mask, tmp_name, vec_els);
191 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
192 : }
193 : else
194 : vect_name = tmp_name;
195 :
196 0 : return vect_name;
197 : }
198 :
199 : /* ARRAY is an array of vectors created by create_vector_array.
200 : Emit code to store SSA_NAME VECT in index N of the array.
201 : The store is part of the vectorization of STMT_INFO. */
202 :
203 : static void
204 0 : write_vector_array (vec_info *vinfo,
205 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
206 : tree vect, tree array, unsigned HOST_WIDE_INT n)
207 : {
208 0 : tree array_ref;
209 0 : gimple *new_stmt;
210 :
211 0 : array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
212 0 : build_int_cst (size_type_node, n),
213 : NULL_TREE, NULL_TREE);
214 :
215 0 : new_stmt = gimple_build_assign (array_ref, vect);
216 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
217 0 : }
218 :
219 : /* PTR is a pointer to an array of type TYPE. Return a representation
220 : of *PTR. The memory reference replaces those in FIRST_DR
221 : (and its group). */
222 :
223 : static tree
224 0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
225 : {
226 0 : tree mem_ref;
227 :
228 0 : mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
229 : /* Arrays have the same alignment as their type. */
230 0 : set_ssa_name_alignment (ptr, TYPE_ALIGN_UNIT (type), 0);
231 0 : return mem_ref;
232 : }
233 :
234 : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
235 : Emit the clobber before *GSI. */
236 :
237 : static void
238 20 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
239 : gimple_stmt_iterator *gsi, tree var)
240 : {
241 20 : tree clobber = build_clobber (TREE_TYPE (var));
242 20 : gimple *new_stmt = gimple_build_assign (var, clobber);
243 20 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
244 20 : }
245 :
246 : /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
247 :
248 : /* Function vect_mark_relevant.
249 :
250 : Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
251 :
252 : static void
253 3331317 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
254 : enum vect_relevant relevant, bool live_p)
255 : {
256 3331317 : enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
257 3331317 : bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
258 :
259 3331317 : if (dump_enabled_p ())
260 172732 : dump_printf_loc (MSG_NOTE, vect_location,
261 : "mark relevant %d, live %d: %G", relevant, live_p,
262 : stmt_info->stmt);
263 :
264 : /* If this stmt is an original stmt in a pattern, we might need to mark its
265 : related pattern stmt instead of the original stmt. However, such stmts
266 : may have their own uses that are not in any pattern, in such cases the
267 : stmt itself should be marked. */
268 3331317 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
269 : {
270 : /* This is the last stmt in a sequence that was detected as a
271 : pattern that can potentially be vectorized. Don't mark the stmt
272 : as relevant/live because it's not going to be vectorized.
273 : Instead mark the pattern-stmt that replaces it. */
274 :
275 258629 : if (dump_enabled_p ())
276 3134 : dump_printf_loc (MSG_NOTE, vect_location,
277 : "last stmt in pattern. don't mark"
278 : " relevant/live.\n");
279 :
280 258629 : stmt_vec_info old_stmt_info = stmt_info;
281 258629 : stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
282 258629 : gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
283 258629 : save_relevant = STMT_VINFO_RELEVANT (stmt_info);
284 258629 : save_live_p = STMT_VINFO_LIVE_P (stmt_info);
285 :
286 258629 : if (live_p && relevant == vect_unused_in_scope)
287 : {
288 119 : if (dump_enabled_p ())
289 10 : dump_printf_loc (MSG_NOTE, vect_location,
290 : "vec_stmt_relevant_p: forcing live pattern stmt "
291 : "relevant.\n");
292 : relevant = vect_used_only_live;
293 : }
294 :
295 258629 : if (dump_enabled_p ())
296 3134 : dump_printf_loc (MSG_NOTE, vect_location,
297 : "mark relevant %d, live %d: %G", relevant, live_p,
298 : stmt_info->stmt);
299 : }
300 :
301 3331317 : STMT_VINFO_LIVE_P (stmt_info) |= live_p;
302 3331317 : if (relevant > STMT_VINFO_RELEVANT (stmt_info))
303 2995778 : STMT_VINFO_RELEVANT (stmt_info) = relevant;
304 :
305 3331317 : if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
306 335539 : && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
307 : {
308 334855 : if (dump_enabled_p ())
309 20912 : dump_printf_loc (MSG_NOTE, vect_location,
310 : "already marked relevant/live.\n");
311 : return;
312 : }
313 :
314 2996462 : worklist->safe_push (stmt_info);
315 : }
316 :
317 :
318 : /* Function is_simple_and_all_uses_invariant
319 :
320 : Return true if STMT_INFO is simple and all uses of it are invariant. */
321 :
322 : bool
323 260911 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
324 : loop_vec_info loop_vinfo)
325 : {
326 260911 : tree op;
327 260911 : ssa_op_iter iter;
328 :
329 260911 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
330 204046 : if (!stmt)
331 : return false;
332 :
333 211512 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
334 : {
335 210709 : enum vect_def_type dt = vect_uninitialized_def;
336 :
337 210709 : if (!vect_is_simple_use (op, loop_vinfo, &dt))
338 : {
339 5515 : if (dump_enabled_p ())
340 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
341 : "use not simple.\n");
342 203243 : return false;
343 : }
344 :
345 205194 : if (dt != vect_external_def && dt != vect_constant_def)
346 : return false;
347 : }
348 : return true;
349 : }
350 :
351 : /* Function vect_stmt_relevant_p.
352 :
353 : Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
354 : is "relevant for vectorization".
355 :
356 : A stmt is considered "relevant for vectorization" if:
357 : - it has uses outside the loop.
358 : - it has vdefs (it alters memory).
359 : - control stmts in the loop (except for the exit condition).
360 :
361 : CHECKME: what other side effects would the vectorizer allow? */
362 :
363 : static bool
364 5393706 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
365 : enum vect_relevant *relevant, bool *live_p)
366 : {
367 5393706 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
368 5393706 : ssa_op_iter op_iter;
369 5393706 : imm_use_iterator imm_iter;
370 5393706 : use_operand_p use_p;
371 5393706 : def_operand_p def_p;
372 :
373 5393706 : *relevant = vect_unused_in_scope;
374 5393706 : *live_p = false;
375 :
376 : /* cond stmt other than loop exit cond. */
377 5393706 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
378 5393706 : if (is_ctrl_stmt (stmt)
379 643012 : && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
380 5638511 : && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
381 242829 : *relevant = vect_used_in_scope;
382 :
383 : /* changing memory. */
384 5393706 : if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
385 4466756 : if (gimple_vdef (stmt_info->stmt)
386 3823744 : && !gimple_clobber_p (stmt_info->stmt))
387 : {
388 382933 : if (dump_enabled_p ())
389 29165 : dump_printf_loc (MSG_NOTE, vect_location,
390 : "vec_stmt_relevant_p: stmt has vdefs.\n");
391 382933 : *relevant = vect_used_in_scope;
392 382933 : if (! STMT_VINFO_DATA_REF (stmt_info)
393 382933 : && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
394 26 : LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
395 : }
396 :
397 : /* uses outside the loop. */
398 15148600 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
399 : {
400 11703740 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
401 : {
402 7342552 : basic_block bb = gimple_bb (USE_STMT (use_p));
403 7342552 : if (!flow_bb_inside_loop_p (loop, bb))
404 : {
405 277464 : if (is_gimple_debug (USE_STMT (use_p)))
406 1187 : continue;
407 :
408 276277 : if (dump_enabled_p ())
409 6367 : dump_printf_loc (MSG_NOTE, vect_location,
410 : "vec_stmt_relevant_p: used out of loop.\n");
411 :
412 : /* We expect all such uses to be in the loop exit phis
413 : (because of loop closed form) */
414 276277 : gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
415 :
416 276277 : *live_p = true;
417 276277 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
418 : }
419 4361188 : }
420 : }
421 :
422 260913 : if (*live_p && *relevant == vect_unused_in_scope
423 5654617 : && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
424 : {
425 260108 : if (dump_enabled_p ())
426 6225 : dump_printf_loc (MSG_NOTE, vect_location,
427 : "vec_stmt_relevant_p: stmt live but not relevant.\n");
428 260108 : *relevant = vect_used_only_live;
429 : }
430 :
431 5393706 : return (*live_p || *relevant);
432 : }
433 :
434 :
435 : /* Function exist_non_indexing_operands_for_use_p
436 :
437 : USE is one of the uses attached to STMT_INFO. Check if USE is
438 : used in STMT_INFO for anything other than indexing an array. */
439 :
440 : static bool
441 4469879 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
442 : {
443 4469879 : tree operand;
444 :
445 : /* USE corresponds to some operand in STMT. If there is no data
446 : reference in STMT, then any operand that corresponds to USE
447 : is not indexing an array. */
448 4469879 : if (!STMT_VINFO_DATA_REF (stmt_info))
449 : return true;
450 :
451 : /* STMT has a data_ref. FORNOW this means that its of one of
452 : the following forms:
453 : -1- ARRAY_REF = var
454 : -2- var = ARRAY_REF
455 : (This should have been verified in analyze_data_refs).
456 :
457 : 'var' in the second case corresponds to a def, not a use,
458 : so USE cannot correspond to any operands that are not used
459 : for array indexing.
460 :
461 : Therefore, all we need to check is if STMT falls into the
462 : first case, and whether var corresponds to USE. */
463 :
464 1524996 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
465 1508127 : if (!assign || !gimple_assign_copy_p (assign))
466 : {
467 818587 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
468 16869 : if (call && gimple_call_internal_p (call))
469 : {
470 16869 : internal_fn ifn = gimple_call_internal_fn (call);
471 16869 : int mask_index = internal_fn_mask_index (ifn);
472 16869 : if (mask_index >= 0
473 16869 : && use == gimple_call_arg (call, mask_index))
474 : return true;
475 11018 : int els_index = internal_fn_else_index (ifn);
476 11018 : if (els_index >= 0
477 11018 : && use == gimple_call_arg (call, els_index))
478 : return true;
479 9624 : int stored_value_index = internal_fn_stored_value_index (ifn);
480 9624 : if (stored_value_index >= 0
481 9624 : && use == gimple_call_arg (call, stored_value_index))
482 : return true;
483 7449 : if (internal_gather_scatter_fn_p (ifn)
484 7449 : && use == gimple_call_arg (call, 1))
485 : return true;
486 : }
487 809167 : return false;
488 : }
489 :
490 706409 : if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
491 : return false;
492 706409 : operand = gimple_assign_rhs1 (assign);
493 706409 : if (TREE_CODE (operand) != SSA_NAME)
494 : return false;
495 :
496 611761 : if (operand == use)
497 288802 : return true;
498 :
499 : return false;
500 : }
501 :
502 :
503 : /*
504 : Function process_use.
505 :
506 : Inputs:
507 : - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
508 : - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
509 : that defined USE. This is done by calling mark_relevant and passing it
510 : the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
511 : - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
512 : be performed.
513 :
514 : Outputs:
515 : Generally, LIVE_P and RELEVANT are used to define the liveness and
516 : relevance info of the DEF_STMT of this USE:
517 : STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
518 : STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
519 : Exceptions:
520 : - case 1: If USE is used only for address computations (e.g. array indexing),
521 : which does not need to be directly vectorized, then the liveness/relevance
522 : of the respective DEF_STMT is left unchanged.
523 : - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
524 : we skip DEF_STMT cause it had already been processed.
525 : - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
526 : "relevant" will be modified accordingly.
527 :
528 : Return true if everything is as expected. Return false otherwise. */
529 :
530 : static opt_result
531 4522785 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
532 : enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
533 : bool force)
534 : {
535 4522785 : stmt_vec_info dstmt_vinfo;
536 4522785 : enum vect_def_type dt;
537 :
538 : /* case 1: we are only interested in uses that need to be vectorized. Uses
539 : that are used for address computation are not considered relevant. */
540 4522785 : if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
541 1226774 : return opt_result::success ();
542 :
543 3296011 : if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
544 36440 : return opt_result::failure_at (stmt_vinfo->stmt,
545 : "not vectorized:"
546 : " unsupported use in stmt.\n");
547 :
548 3259571 : if (!dstmt_vinfo)
549 618114 : return opt_result::success ();
550 :
551 2641457 : basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
552 2641457 : basic_block bb = gimple_bb (stmt_vinfo->stmt);
553 :
554 : /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
555 : We have to force the stmt live since the epilogue loop needs it to
556 : continue computing the reduction. */
557 2641457 : if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
558 278083 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
559 86022 : && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
560 86022 : && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
561 2727479 : && bb->loop_father == def_bb->loop_father)
562 : {
563 86022 : if (dump_enabled_p ())
564 4076 : dump_printf_loc (MSG_NOTE, vect_location,
565 : "reduc-stmt defining reduc-phi in the same nest.\n");
566 86022 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
567 86022 : return opt_result::success ();
568 : }
569 :
570 : /* case 3a: outer-loop stmt defining an inner-loop stmt:
571 : outer-loop-header-bb:
572 : d = dstmt_vinfo
573 : inner-loop:
574 : stmt # use (d)
575 : outer-loop-tail-bb:
576 : ... */
577 2555435 : if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
578 : {
579 2141 : if (dump_enabled_p ())
580 324 : dump_printf_loc (MSG_NOTE, vect_location,
581 : "outer-loop def-stmt defining inner-loop stmt.\n");
582 :
583 2141 : switch (relevant)
584 : {
585 0 : case vect_unused_in_scope:
586 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
587 : vect_used_in_scope : vect_unused_in_scope;
588 : break;
589 :
590 767 : case vect_used_in_outer_by_reduction:
591 767 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
592 : relevant = vect_used_by_reduction;
593 : break;
594 :
595 1088 : case vect_used_in_outer:
596 1088 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
597 : relevant = vect_used_in_scope;
598 : break;
599 :
600 : case vect_used_in_scope:
601 : break;
602 :
603 0 : default:
604 0 : gcc_unreachable ();
605 : }
606 : }
607 :
608 : /* case 3b: inner-loop stmt defining an outer-loop stmt:
609 : outer-loop-header-bb:
610 : ...
611 : inner-loop:
612 : d = dstmt_vinfo
613 : outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
614 : stmt # use (d) */
615 2553294 : else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
616 : {
617 2073 : if (dump_enabled_p ())
618 629 : dump_printf_loc (MSG_NOTE, vect_location,
619 : "inner-loop def-stmt defining outer-loop stmt.\n");
620 :
621 2073 : switch (relevant)
622 : {
623 0 : case vect_unused_in_scope:
624 2369599 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
625 0 : || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
626 : vect_used_in_outer_by_reduction : vect_unused_in_scope;
627 : break;
628 :
629 : case vect_used_by_reduction:
630 : case vect_used_only_live:
631 : relevant = vect_used_in_outer_by_reduction;
632 : break;
633 :
634 : case vect_used_in_scope:
635 2369599 : relevant = vect_used_in_outer;
636 : break;
637 :
638 0 : default:
639 0 : gcc_unreachable ();
640 : }
641 : }
642 : /* We are also not interested in uses on loop PHI backedges that are
643 : inductions. Otherwise we'll needlessly vectorize the IV increment
644 : and cause hybrid SLP for SLP inductions. */
645 2551221 : else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
646 188847 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
647 2737057 : && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
648 : loop_latch_edge (bb->loop_father))
649 : == use))
650 : {
651 185836 : if (dump_enabled_p ())
652 5307 : dump_printf_loc (MSG_NOTE, vect_location,
653 : "induction value on backedge.\n");
654 185836 : return opt_result::success ();
655 : }
656 :
657 2369599 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
658 2369599 : return opt_result::success ();
659 : }
660 :
661 :
662 : /* Function vect_mark_stmts_to_be_vectorized.
663 :
664 : Not all stmts in the loop need to be vectorized. For example:
665 :
666 : for i...
667 : for j...
668 : 1. T0 = i + j
669 : 2. T1 = a[T0]
670 :
671 : 3. j = j + 1
672 :
673 : Stmt 1 and 3 do not need to be vectorized, because loop control and
674 : addressing of vectorized data-refs are handled differently.
675 :
676 : This pass detects such stmts. */
677 :
678 : opt_result
679 456698 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
680 : {
681 456698 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
682 456698 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
683 456698 : unsigned int nbbs = loop->num_nodes;
684 456698 : gimple_stmt_iterator si;
685 456698 : unsigned int i;
686 456698 : basic_block bb;
687 456698 : bool live_p;
688 456698 : enum vect_relevant relevant;
689 :
690 456698 : DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
691 :
692 456698 : auto_vec<stmt_vec_info, 64> worklist;
693 :
694 : /* 1. Init worklist. */
695 1549252 : for (i = 0; i < nbbs; i++)
696 : {
697 1103676 : bb = bbs[i];
698 2258930 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
699 : {
700 2332462 : if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
701 239281 : continue;
702 926950 : stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
703 926950 : if (dump_enabled_p ())
704 43771 : dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
705 : phi_info->stmt);
706 :
707 926950 : if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
708 : {
709 47985 : if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
710 10977 : return opt_result::failure_at
711 10977 : (*si, "not vectorized: unhandled relevant PHI: %G", *si);
712 37008 : vect_mark_relevant (&worklist, phi_info, relevant, live_p);
713 : }
714 : }
715 8782171 : for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
716 : {
717 7689617 : gimple *stmt = gsi_stmt (si);
718 7689617 : if (is_gimple_debug (stmt))
719 3222716 : continue;
720 4466901 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
721 4466901 : if (dump_enabled_p ())
722 236106 : dump_printf_loc (MSG_NOTE, vect_location,
723 : "init: stmt relevant? %G", stmt);
724 :
725 4466901 : if (gimple_get_lhs (stmt) == NULL_TREE
726 649271 : && !is_a <gcond *> (stmt)
727 4473160 : && !is_a <gcall *> (stmt))
728 145 : return opt_result::failure_at
729 145 : (stmt, "not vectorized: irregular stmt: %G", stmt);
730 :
731 4466756 : if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
732 838688 : vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
733 : }
734 : }
735 :
736 : /* 2. Process_worklist */
737 3312417 : while (worklist.length () > 0)
738 : {
739 2903283 : use_operand_p use_p;
740 2903283 : ssa_op_iter iter;
741 :
742 2903283 : stmt_vec_info stmt_vinfo = worklist.pop ();
743 2903283 : if (dump_enabled_p ())
744 151147 : dump_printf_loc (MSG_NOTE, vect_location,
745 : "worklist: examine stmt: %G", stmt_vinfo->stmt);
746 :
747 : /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
748 : (DEF_STMT) as relevant/irrelevant according to the relevance property
749 : of STMT. */
750 2903283 : relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
751 :
752 : /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
753 : propagated as is to the DEF_STMTs of its USEs.
754 :
755 : One exception is when STMT has been identified as defining a reduction
756 : variable; in this case we set the relevance to vect_used_by_reduction.
757 : This is because we distinguish between two kinds of relevant stmts -
758 : those that are used by a reduction computation, and those that are
759 : (also) used by a regular computation. This allows us later on to
760 : identify stmts that are used solely by a reduction, and therefore the
761 : order of the results that they produce does not have to be kept. */
762 :
763 2903283 : switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
764 : {
765 174169 : case vect_reduction_def:
766 174169 : gcc_assert (relevant != vect_unused_in_scope);
767 174169 : if (relevant != vect_unused_in_scope
768 174169 : && relevant != vect_used_in_scope
769 174169 : && relevant != vect_used_by_reduction
770 174169 : && relevant != vect_used_only_live)
771 0 : return opt_result::failure_at
772 0 : (stmt_vinfo->stmt, "unsupported use of reduction.\n");
773 : break;
774 :
775 2092 : case vect_nested_cycle:
776 2092 : if (relevant != vect_unused_in_scope
777 2092 : && relevant != vect_used_in_outer_by_reduction
778 1497 : && relevant != vect_used_in_outer)
779 2 : return opt_result::failure_at
780 2 : (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
781 : break;
782 :
783 1209 : case vect_double_reduction_def:
784 1209 : if (relevant != vect_unused_in_scope
785 1209 : && relevant != vect_used_by_reduction
786 409 : && relevant != vect_used_only_live)
787 0 : return opt_result::failure_at
788 0 : (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
789 : break;
790 :
791 : default:
792 : break;
793 : }
794 :
795 2903281 : if (is_pattern_stmt_p (stmt_vinfo))
796 : {
797 : /* Pattern statements are not inserted into the code, so
798 : FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
799 : have to scan the RHS or function arguments instead. */
800 661357 : if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
801 : {
802 428127 : enum tree_code rhs_code = gimple_assign_rhs_code (assign);
803 428127 : tree op = gimple_assign_rhs1 (assign);
804 :
805 428127 : i = 1;
806 428127 : if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
807 : {
808 0 : opt_result res
809 0 : = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
810 : loop_vinfo, relevant, &worklist, false);
811 0 : if (!res)
812 0 : return res;
813 0 : res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
814 : loop_vinfo, relevant, &worklist, false);
815 0 : if (!res)
816 0 : return res;
817 428127 : i = 2;
818 : }
819 1233634 : for (; i < gimple_num_ops (assign); i++)
820 : {
821 809538 : op = gimple_op (assign, i);
822 809538 : if (TREE_CODE (op) == SSA_NAME)
823 : {
824 615045 : opt_result res
825 615045 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
826 : &worklist, false);
827 615045 : if (!res)
828 4031 : return res;
829 : }
830 : }
831 : }
832 233230 : else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
833 : {
834 226102 : tree_code rhs_code = gimple_cond_code (cond);
835 226102 : gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
836 226102 : opt_result res
837 226102 : = process_use (stmt_vinfo, gimple_cond_lhs (cond),
838 : loop_vinfo, relevant, &worklist, false);
839 226102 : if (!res)
840 36442 : return res;
841 226102 : res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
842 : loop_vinfo, relevant, &worklist, false);
843 226102 : if (!res)
844 0 : return res;
845 : }
846 7128 : else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
847 : {
848 33335 : for (i = 0; i < gimple_call_num_args (call); i++)
849 : {
850 26207 : tree arg = gimple_call_arg (call, i);
851 26207 : opt_result res
852 26207 : = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
853 : &worklist, false);
854 26207 : if (!res)
855 0 : return res;
856 : }
857 : }
858 : else
859 0 : gcc_unreachable ();
860 : }
861 : else
862 7841997 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
863 : {
864 3376423 : tree op = USE_FROM_PTR (use_p);
865 3376423 : opt_result res
866 3376423 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
867 : &worklist, false);
868 3376423 : if (!res)
869 18274 : return res;
870 : }
871 :
872 2880976 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
873 : {
874 52906 : gather_scatter_info gs_info;
875 52906 : if (!vect_check_gather_scatter (stmt_vinfo,
876 : STMT_VINFO_VECTYPE (stmt_vinfo),
877 : loop_vinfo, &gs_info))
878 0 : gcc_unreachable ();
879 52906 : opt_result res
880 52906 : = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
881 : &worklist, true);
882 52906 : if (!res)
883 : {
884 14135 : if (fatal)
885 14135 : *fatal = false;
886 14135 : return res;
887 : }
888 : }
889 : } /* while worklist */
890 :
891 409134 : return opt_result::success ();
892 456698 : }
893 :
894 : /* Function vect_model_simple_cost.
895 :
896 : Models cost for simple operations, i.e. those that only emit N operations
897 : of the same KIND. */
898 :
899 : static void
900 676982 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
901 : stmt_vector_for_cost *cost_vec,
902 : vect_cost_for_stmt kind = vector_stmt)
903 : {
904 676982 : int inside_cost = 0, prologue_cost = 0;
905 :
906 676982 : gcc_assert (cost_vec != NULL);
907 :
908 676982 : n *= vect_get_num_copies (vinfo, node);
909 :
910 : /* Pass the inside-of-loop statements to the target-specific cost model. */
911 676982 : inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
912 :
913 676982 : if (dump_enabled_p ())
914 36163 : dump_printf_loc (MSG_NOTE, vect_location,
915 : "vect_model_simple_cost: inside_cost = %d, "
916 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
917 676982 : }
918 :
919 :
920 : /* Model cost for type demotion and promotion operations. PWR is
921 : normally zero for single-step promotions and demotions. It will be
922 : one if two-step promotion/demotion is required, and so on. NCOPIES
923 : is the number of vector results (and thus number of instructions)
924 : for the narrowest end of the operation chain. Each additional
925 : step doubles the number of instructions required. If WIDEN_ARITH
926 : is true the stmt is doing widening arithmetic. */
927 :
928 : static void
929 70426 : vect_model_promotion_demotion_cost (slp_tree slp_node,
930 : unsigned int ncopies, int pwr,
931 : stmt_vector_for_cost *cost_vec,
932 : bool widen_arith)
933 : {
934 70426 : int i;
935 70426 : int inside_cost = 0, prologue_cost = 0;
936 :
937 163917 : for (i = 0; i < pwr + 1; i++)
938 : {
939 185047 : inside_cost += record_stmt_cost (cost_vec, ncopies,
940 : widen_arith
941 : ? vector_stmt : vec_promote_demote,
942 : slp_node, 0, vect_body);
943 93491 : ncopies *= 2;
944 : }
945 :
946 70426 : if (dump_enabled_p ())
947 6530 : dump_printf_loc (MSG_NOTE, vect_location,
948 : "vect_model_promotion_demotion_cost: inside_cost = %d, "
949 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
950 70426 : }
951 :
952 : /* Returns true if the current function returns DECL. */
953 :
954 : static bool
955 572662 : cfun_returns (tree decl)
956 : {
957 572662 : edge_iterator ei;
958 572662 : edge e;
959 1127797 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
960 : {
961 1133916 : greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
962 566958 : if (!ret)
963 0 : continue;
964 566958 : if (gimple_return_retval (ret) == decl)
965 : return true;
966 : /* We often end up with an aggregate copy to the result decl,
967 : handle that case as well. First skip intermediate clobbers
968 : though. */
969 : gimple *def = ret;
970 1707910 : do
971 : {
972 3415820 : def = SSA_NAME_DEF_STMT (gimple_vuse (def));
973 : }
974 1707910 : while (gimple_clobber_p (def));
975 555822 : if (is_a <gassign *> (def)
976 63746 : && gimple_assign_lhs (def) == gimple_return_retval (ret)
977 562762 : && gimple_assign_rhs1 (def) == decl)
978 : return true;
979 : }
980 : return false;
981 : }
982 :
983 : /* Calculate cost of DR's memory access. */
984 : void
985 1048810 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
986 : int ncopies, dr_alignment_support alignment_support_scheme,
987 : int misalignment,
988 : unsigned int *inside_cost,
989 : stmt_vector_for_cost *body_cost_vec)
990 : {
991 1048810 : tree vectype
992 1048810 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
993 1048810 : switch (alignment_support_scheme)
994 : {
995 576875 : case dr_aligned:
996 576875 : {
997 576875 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
998 : vector_store, stmt_info, slp_node,
999 : vectype, 0, vect_body);
1000 :
1001 576875 : if (dump_enabled_p ())
1002 15580 : dump_printf_loc (MSG_NOTE, vect_location,
1003 : "vect_model_store_cost: aligned.\n");
1004 : break;
1005 : }
1006 :
1007 471935 : case dr_unaligned_supported:
1008 471935 : {
1009 : /* Here, we assign an additional cost for the unaligned store. */
1010 471935 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1011 : unaligned_store, stmt_info, slp_node,
1012 : vectype, misalignment, vect_body);
1013 471935 : if (dump_enabled_p ())
1014 14187 : dump_printf_loc (MSG_NOTE, vect_location,
1015 : "vect_model_store_cost: unaligned supported by "
1016 : "hardware.\n");
1017 : break;
1018 : }
1019 :
1020 0 : case dr_unaligned_unsupported:
1021 0 : {
1022 0 : *inside_cost = VECT_MAX_COST;
1023 :
1024 0 : if (dump_enabled_p ())
1025 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1026 : "vect_model_store_cost: unsupported access.\n");
1027 : break;
1028 : }
1029 :
1030 0 : default:
1031 0 : gcc_unreachable ();
1032 : }
1033 1048810 : }
1034 :
1035 : /* Calculate cost of DR's memory access. */
1036 : void
1037 996373 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
1038 : int ncopies, dr_alignment_support alignment_support_scheme,
1039 : int misalignment,
1040 : bool add_realign_cost, unsigned int *inside_cost,
1041 : unsigned int *prologue_cost,
1042 : stmt_vector_for_cost *prologue_cost_vec,
1043 : stmt_vector_for_cost *body_cost_vec,
1044 : bool record_prologue_costs)
1045 : {
1046 996373 : tree vectype
1047 996373 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
1048 996373 : switch (alignment_support_scheme)
1049 : {
1050 572113 : case dr_aligned:
1051 572113 : {
1052 572113 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1053 : stmt_info, slp_node, vectype,
1054 : 0, vect_body);
1055 :
1056 572113 : if (dump_enabled_p ())
1057 20333 : dump_printf_loc (MSG_NOTE, vect_location,
1058 : "vect_model_load_cost: aligned.\n");
1059 :
1060 : break;
1061 : }
1062 363096 : case dr_unaligned_supported:
1063 363096 : {
1064 : /* Here, we assign an additional cost for the unaligned load. */
1065 363096 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1066 : unaligned_load, stmt_info, slp_node,
1067 : vectype, misalignment, vect_body);
1068 :
1069 363096 : if (dump_enabled_p ())
1070 24546 : dump_printf_loc (MSG_NOTE, vect_location,
1071 : "vect_model_load_cost: unaligned supported by "
1072 : "hardware.\n");
1073 :
1074 : break;
1075 : }
1076 0 : case dr_explicit_realign:
1077 0 : {
1078 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1079 : vector_load, stmt_info, slp_node,
1080 : vectype, 0, vect_body);
1081 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1082 : vec_perm, stmt_info, slp_node,
1083 : vectype, 0, vect_body);
1084 :
1085 : /* FIXME: If the misalignment remains fixed across the iterations of
1086 : the containing loop, the following cost should be added to the
1087 : prologue costs. */
1088 0 : if (targetm.vectorize.builtin_mask_for_load)
1089 0 : *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1090 : stmt_info, slp_node, vectype,
1091 : 0, vect_body);
1092 :
1093 0 : if (dump_enabled_p ())
1094 0 : dump_printf_loc (MSG_NOTE, vect_location,
1095 : "vect_model_load_cost: explicit realign\n");
1096 :
1097 : break;
1098 : }
1099 0 : case dr_explicit_realign_optimized:
1100 0 : {
1101 0 : if (dump_enabled_p ())
1102 0 : dump_printf_loc (MSG_NOTE, vect_location,
1103 : "vect_model_load_cost: unaligned software "
1104 : "pipelined.\n");
1105 :
1106 : /* Unaligned software pipeline has a load of an address, an initial
1107 : load, and possibly a mask operation to "prime" the loop. However,
1108 : if this is an access in a group of loads, which provide grouped
1109 : access, then the above cost should only be considered for one
1110 : access in the group. Inside the loop, there is a load op
1111 : and a realignment op. */
1112 :
1113 0 : if (add_realign_cost && record_prologue_costs)
1114 : {
1115 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1116 : vector_stmt, stmt_info,
1117 : slp_node, vectype,
1118 : 0, vect_prologue);
1119 0 : if (targetm.vectorize.builtin_mask_for_load)
1120 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1121 : vector_stmt, stmt_info,
1122 : slp_node, vectype,
1123 : 0, vect_prologue);
1124 : }
1125 :
1126 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1127 : stmt_info, slp_node, vectype,
1128 : 0, vect_body);
1129 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1130 : stmt_info, slp_node, vectype,
1131 : 0, vect_body);
1132 :
1133 0 : if (dump_enabled_p ())
1134 0 : dump_printf_loc (MSG_NOTE, vect_location,
1135 : "vect_model_load_cost: explicit realign optimized"
1136 : "\n");
1137 :
1138 : break;
1139 : }
1140 :
1141 61164 : case dr_unaligned_unsupported:
1142 61164 : {
1143 61164 : *inside_cost = VECT_MAX_COST;
1144 :
1145 61164 : if (dump_enabled_p ())
1146 116 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1147 : "vect_model_load_cost: unsupported access.\n");
1148 : break;
1149 : }
1150 :
1151 0 : default:
1152 0 : gcc_unreachable ();
1153 : }
1154 996373 : }
1155 :
1156 : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1157 : the loop preheader for the vectorized stmt STMT_VINFO. */
1158 :
1159 : static void
1160 6760 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
1161 : gimple_stmt_iterator *gsi)
1162 : {
1163 6760 : if (gsi)
1164 3480 : vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
1165 : else
1166 3280 : vinfo->insert_on_entry (stmt_vinfo, new_stmt);
1167 :
1168 6760 : if (dump_enabled_p ())
1169 1860 : dump_printf_loc (MSG_NOTE, vect_location,
1170 : "created new init_stmt: %G", new_stmt);
1171 6760 : }
1172 :
1173 : /* Function vect_init_vector.
1174 :
1175 : Insert a new stmt (INIT_STMT) that initializes a new variable of type
1176 : TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1177 : vector type a vector with all elements equal to VAL is created first.
1178 : Place the initialization at GSI if it is not NULL. Otherwise, place the
1179 : initialization at the loop preheader.
1180 : Return the DEF of INIT_STMT.
1181 : It will be used in the vectorization of STMT_INFO. */
1182 :
1183 : tree
1184 5042 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
1185 : gimple_stmt_iterator *gsi)
1186 : {
1187 5042 : gimple *init_stmt;
1188 5042 : tree new_temp;
1189 :
1190 : /* We abuse this function to push sth to a SSA name with initial 'val'. */
1191 5042 : if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1192 : {
1193 1316 : gcc_assert (VECTOR_TYPE_P (type));
1194 1316 : if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1195 : {
1196 : /* Scalar boolean value should be transformed into
1197 : all zeros or all ones value before building a vector. */
1198 11 : if (VECTOR_BOOLEAN_TYPE_P (type))
1199 : {
1200 3 : tree true_val = build_all_ones_cst (TREE_TYPE (type));
1201 3 : tree false_val = build_zero_cst (TREE_TYPE (type));
1202 :
1203 3 : if (CONSTANT_CLASS_P (val))
1204 0 : val = integer_zerop (val) ? false_val : true_val;
1205 : else
1206 : {
1207 3 : new_temp = make_ssa_name (TREE_TYPE (type));
1208 3 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1209 : val, true_val, false_val);
1210 3 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1211 3 : val = new_temp;
1212 : }
1213 : }
1214 : else
1215 : {
1216 8 : gimple_seq stmts = NULL;
1217 8 : if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1218 8 : val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
1219 8 : TREE_TYPE (type), val);
1220 : else
1221 : /* ??? Condition vectorization expects us to do
1222 : promotion of invariant/external defs. */
1223 0 : val = gimple_convert (&stmts, TREE_TYPE (type), val);
1224 16 : for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
1225 16 : !gsi_end_p (gsi2); )
1226 : {
1227 8 : init_stmt = gsi_stmt (gsi2);
1228 8 : gsi_remove (&gsi2, false);
1229 8 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1230 : }
1231 : }
1232 : }
1233 1316 : val = build_vector_from_val (type, val);
1234 : }
1235 :
1236 5042 : new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1237 5042 : init_stmt = gimple_build_assign (new_temp, val);
1238 5042 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1239 5042 : return new_temp;
1240 : }
1241 :
1242 :
1243 : /* Get vectorized definitions for OP0 and OP1. */
1244 :
1245 : void
1246 194630 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
1247 : bool op0, vec<tree> *vec_oprnds0,
1248 : bool op1, vec<tree> *vec_oprnds1,
1249 : bool op2, vec<tree> *vec_oprnds2,
1250 : bool op3, vec<tree> *vec_oprnds3)
1251 : {
1252 194630 : if (op0)
1253 193022 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
1254 194630 : if (op1)
1255 143175 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
1256 194630 : if (op2)
1257 9475 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
1258 194630 : if (op3)
1259 0 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
1260 194630 : }
1261 :
1262 : /* Helper function called by vect_finish_replace_stmt and
1263 : vect_finish_stmt_generation. Set the location of the new
1264 : statement and create and return a stmt_vec_info for it. */
1265 :
1266 : static void
1267 1436282 : vect_finish_stmt_generation_1 (vec_info *,
1268 : stmt_vec_info stmt_info, gimple *vec_stmt)
1269 : {
1270 1436282 : if (dump_enabled_p ())
1271 151648 : dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
1272 :
1273 1436282 : if (stmt_info)
1274 : {
1275 1407550 : gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
1276 :
1277 : /* While EH edges will generally prevent vectorization, stmt might
1278 : e.g. be in a must-not-throw region. Ensure newly created stmts
1279 : that could throw are part of the same region. */
1280 1407550 : int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
1281 1407550 : if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
1282 48 : add_stmt_to_eh_lp (vec_stmt, lp_nr);
1283 : }
1284 : else
1285 28732 : gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
1286 1436282 : }
1287 :
1288 : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1289 : which sets the same scalar result as STMT_INFO did. Create and return a
1290 : stmt_vec_info for VEC_STMT. */
1291 :
1292 : void
1293 977 : vect_finish_replace_stmt (vec_info *vinfo,
1294 : stmt_vec_info stmt_info, gimple *vec_stmt)
1295 : {
1296 977 : gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
1297 977 : gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
1298 :
1299 977 : gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
1300 977 : gsi_replace (&gsi, vec_stmt, true);
1301 :
1302 977 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1303 977 : }
1304 :
1305 : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1306 : before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1307 :
1308 : void
1309 1435305 : vect_finish_stmt_generation (vec_info *vinfo,
1310 : stmt_vec_info stmt_info, gimple *vec_stmt,
1311 : gimple_stmt_iterator *gsi)
1312 : {
1313 1435305 : gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
1314 :
1315 1435305 : if (!gsi_end_p (*gsi)
1316 2869665 : && gimple_has_mem_ops (vec_stmt))
1317 : {
1318 1434360 : gimple *at_stmt = gsi_stmt (*gsi);
1319 1434360 : tree vuse = gimple_vuse (at_stmt);
1320 1428099 : if (vuse && TREE_CODE (vuse) == SSA_NAME)
1321 : {
1322 1281663 : tree vdef = gimple_vdef (at_stmt);
1323 1281663 : gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1324 1281663 : gimple_set_modified (vec_stmt, true);
1325 : /* If we have an SSA vuse and insert a store, update virtual
1326 : SSA form to avoid triggering the renamer. Do so only
1327 : if we can easily see all uses - which is what almost always
1328 : happens with the way vectorized stmts are inserted. */
1329 759307 : if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1330 2040934 : && ((is_gimple_assign (vec_stmt)
1331 758417 : && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1332 64780 : || (is_gimple_call (vec_stmt)
1333 854 : && (!(gimple_call_flags (vec_stmt)
1334 854 : & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
1335 3 : || (gimple_call_lhs (vec_stmt)
1336 3 : && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
1337 : {
1338 695342 : tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1339 695342 : gimple_set_vdef (vec_stmt, new_vdef);
1340 695342 : SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1341 : }
1342 : }
1343 : }
1344 1435305 : gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1345 1435305 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1346 1435305 : }
1347 :
1348 : /* We want to vectorize a call to combined function CFN with function
1349 : decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1350 : as the types of all inputs. Check whether this is possible using
1351 : an internal function, returning its code if so or IFN_LAST if not. */
1352 :
1353 : static internal_fn
1354 18575 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
1355 : tree vectype_out, tree vectype_in)
1356 : {
1357 18575 : internal_fn ifn;
1358 18575 : if (internal_fn_p (cfn))
1359 14728 : ifn = as_internal_fn (cfn);
1360 : else
1361 3847 : ifn = associated_internal_fn (fndecl);
1362 18575 : if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1363 : {
1364 14759 : const direct_internal_fn_info &info = direct_internal_fn (ifn);
1365 14759 : if (info.vectorizable)
1366 : {
1367 14759 : bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
1368 14759 : tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1369 14759 : tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1370 :
1371 : /* The type size of both the vectype_in and vectype_out should be
1372 : exactly the same when vectype_out isn't participating the optab.
1373 : While there is no restriction for type size when vectype_out
1374 : is part of the optab query. */
1375 14759 : if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
1376 18575 : return IFN_LAST;
1377 :
1378 14722 : if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1379 : OPTIMIZE_FOR_SPEED))
1380 8304 : return ifn;
1381 : }
1382 : }
1383 : return IFN_LAST;
1384 : }
1385 :
1386 :
1387 : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
1388 : gimple_stmt_iterator *);
1389 :
1390 : /* Check whether a load or store statement in the loop described by
1391 : LOOP_VINFO is possible in a loop using partial vectors. This is
1392 : testing whether the vectorizer pass has the appropriate support,
1393 : as well as whether the target does.
1394 :
1395 : VLS_TYPE says whether the statement is a load or store and VECTYPE
1396 : is the type of the vector being loaded or stored. SLP_NODE is the SLP
1397 : node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1398 : says how the load or store is going to be implemented and GROUP_SIZE
1399 : is the number of load or store statements in the containing group.
1400 : If the access is a gather load or scatter store, GS_INFO describes
1401 : its arguments. If the load or store is conditional, SCALAR_MASK is the
1402 : condition under which it occurs.
1403 :
1404 : Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1405 : vectors is not supported, otherwise record the required rgroup control
1406 : types.
1407 :
1408 : If partial vectors can be used and ELSVALS is nonzero the supported
1409 : else values will be added to the vector ELSVALS points to. */
1410 :
1411 : static void
1412 304212 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
1413 : slp_tree slp_node,
1414 : vec_load_store_type vls_type,
1415 : int group_size,
1416 : vect_load_store_data *ls,
1417 : slp_tree mask_node,
1418 : vec<int> *elsvals = nullptr)
1419 : {
1420 304212 : vect_memory_access_type memory_access_type = ls->memory_access_type;
1421 :
1422 : /* Invariant loads need no special support. */
1423 304212 : if (memory_access_type == VMAT_INVARIANT)
1424 30465 : return;
1425 :
1426 : /* Figure whether the mask is uniform. scalar_mask is used to
1427 : populate the scalar_cond_masked_set. */
1428 302987 : tree scalar_mask = NULL_TREE;
1429 302987 : if (mask_node)
1430 4952 : for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
1431 : {
1432 2507 : tree def = vect_get_slp_scalar_def (mask_node, i);
1433 2507 : if (!def
1434 2507 : || (scalar_mask && def != scalar_mask))
1435 : {
1436 : scalar_mask = NULL;
1437 : break;
1438 : }
1439 : else
1440 2486 : scalar_mask = def;
1441 : }
1442 :
1443 302987 : unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
1444 302987 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1445 302987 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
1446 302987 : machine_mode vecmode = TYPE_MODE (vectype);
1447 302987 : bool is_load = (vls_type == VLS_LOAD);
1448 302987 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
1449 : {
1450 0 : nvectors /= group_size;
1451 0 : internal_fn ifn
1452 0 : = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
1453 : elsvals)
1454 0 : : vect_store_lanes_supported (vectype, group_size, true));
1455 0 : if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
1456 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1457 0 : else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
1458 0 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1459 : scalar_mask);
1460 : else
1461 : {
1462 0 : if (dump_enabled_p ())
1463 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1464 : "can't operate on partial vectors because"
1465 : " the target doesn't have an appropriate"
1466 : " load/store-lanes instruction.\n");
1467 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1468 : }
1469 : return;
1470 : }
1471 :
1472 302987 : if (mat_gather_scatter_p (memory_access_type))
1473 : {
1474 1633 : internal_fn ifn = (is_load
1475 1633 : ? IFN_MASK_GATHER_LOAD
1476 : : IFN_MASK_SCATTER_STORE);
1477 342 : internal_fn len_ifn = (is_load
1478 : ? IFN_MASK_LEN_GATHER_LOAD
1479 : : IFN_MASK_LEN_SCATTER_STORE);
1480 1633 : stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
1481 1633 : tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
1482 1633 : ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
1483 1633 : : ls->strided_offset_vectype);
1484 1633 : tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
1485 1633 : int scale = SLP_TREE_GS_SCALE (slp_node);
1486 :
1487 : /* The following "supported" checks just verify what we established in
1488 : get_load_store_type and don't try different offset types.
1489 : Therefore, off_vectype must be a supported offset type. In case
1490 : we chose a different one use this instead. */
1491 1633 : if (ls->supported_offset_vectype)
1492 0 : off_vectype = ls->supported_offset_vectype;
1493 : /* Same for scale. */
1494 1633 : if (ls->supported_scale)
1495 0 : scale = ls->supported_scale;
1496 :
1497 1633 : if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
1498 : memory_type,
1499 : off_vectype, scale,
1500 : elsvals))
1501 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1502 1633 : else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
1503 : memory_type,
1504 : off_vectype, scale,
1505 : elsvals)
1506 1633 : || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
1507 567 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1508 : scalar_mask);
1509 : else
1510 : {
1511 1066 : if (dump_enabled_p ())
1512 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1513 : "can't operate on partial vectors because"
1514 : " the target doesn't have an appropriate"
1515 : " gather load or scatter store instruction.\n");
1516 1066 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1517 : }
1518 : return;
1519 : }
1520 :
1521 301354 : if (memory_access_type != VMAT_CONTIGUOUS)
1522 : {
1523 : /* Element X of the data must come from iteration i * VF + X of the
1524 : scalar loop. We need more work to support other mappings. */
1525 27386 : if (dump_enabled_p ())
1526 758 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1527 : "can't operate on partial vectors because an"
1528 : " access isn't contiguous.\n");
1529 27386 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1530 27386 : return;
1531 : }
1532 :
1533 273968 : if (!VECTOR_MODE_P (vecmode))
1534 : {
1535 0 : if (dump_enabled_p ())
1536 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1537 : "can't operate on partial vectors when emulating"
1538 : " vector operations.\n");
1539 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1540 0 : return;
1541 : }
1542 :
1543 273968 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1544 273968 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1545 :
1546 : /* For permuted SLP loads the group accessed in one vector iteration
1547 : might not fill a whole number of vectors, for example when the
1548 : load permutation duplicates elements. The loop masks and lengths
1549 : are computed based on NVECTORS * NUNITS / VF scalars per iteration
1550 : which would then enable more elements than the scalar loop accesses. */
1551 273968 : if (!constant_multiple_p (group_size * vf, nunits, &nvectors))
1552 : {
1553 221 : if (dump_enabled_p ())
1554 75 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1555 : "can't operate on partial vectors because the"
1556 : " access does not cover a whole number of"
1557 : " vectors.\n");
1558 221 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1559 221 : return;
1560 : }
1561 :
1562 273747 : machine_mode mask_mode;
1563 273747 : machine_mode vmode;
1564 273747 : bool using_partial_vectors_p = false;
1565 273747 : if (get_len_load_store_mode
1566 273747 : (vecmode, is_load, nullptr, elsvals).exists (&vmode))
1567 : {
1568 0 : unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
1569 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
1570 0 : using_partial_vectors_p = true;
1571 : }
1572 273747 : else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
1573 273747 : && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
1574 : elsvals))
1575 : {
1576 92435 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
1577 92435 : using_partial_vectors_p = true;
1578 : }
1579 :
1580 92435 : if (!using_partial_vectors_p)
1581 : {
1582 181312 : if (dump_enabled_p ())
1583 12883 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1584 : "can't operate on partial vectors because the"
1585 : " target doesn't have the appropriate partial"
1586 : " vectorization load or store.\n");
1587 181312 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1588 : }
1589 : }
1590 :
1591 : /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1592 : form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1593 : that needs to be applied to all loads and stores in a vectorized loop.
1594 : Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1595 : otherwise return VEC_MASK & LOOP_MASK.
1596 :
1597 : MASK_TYPE is the type of both masks. If new statements are needed,
1598 : insert them before GSI. */
1599 :
1600 : tree
1601 1629 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
1602 : tree vec_mask, gimple_stmt_iterator *gsi)
1603 : {
1604 1629 : gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1605 1629 : if (!loop_mask)
1606 : return vec_mask;
1607 :
1608 145 : gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1609 :
1610 145 : if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
1611 : return vec_mask;
1612 :
1613 145 : tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1614 145 : gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1615 : vec_mask, loop_mask);
1616 :
1617 145 : gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1618 145 : return and_res;
1619 : }
1620 :
1621 : /* Determine whether we can use a gather load or scatter store to vectorize
1622 : strided load or store STMT_INFO by truncating the current offset to a
1623 : smaller width. We need to be able to construct an offset vector:
1624 :
1625 : { 0, X, X*2, X*3, ... }
1626 :
1627 : without loss of precision, where X is STMT_INFO's DR_STEP.
1628 :
1629 : Return true if this is possible, describing the gather load or scatter
1630 : store in GS_INFO. MASKED_P is true if the load or store is conditional.
1631 :
1632 : If we can use gather/scatter and ELSVALS is nonzero the supported
1633 : else values will be stored in the vector ELSVALS points to. */
1634 :
1635 : static bool
1636 65671 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
1637 : loop_vec_info loop_vinfo, bool masked_p,
1638 : gather_scatter_info *gs_info,
1639 : vec<int> *elsvals)
1640 : {
1641 65671 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1642 65671 : data_reference *dr = dr_info->dr;
1643 65671 : tree step = DR_STEP (dr);
1644 65671 : if (TREE_CODE (step) != INTEGER_CST)
1645 : {
1646 : /* ??? Perhaps we could use range information here? */
1647 29644 : if (dump_enabled_p ())
1648 205 : dump_printf_loc (MSG_NOTE, vect_location,
1649 : "cannot truncate variable step.\n");
1650 : return false;
1651 : }
1652 :
1653 : /* Get the number of bits in an element. */
1654 36027 : scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1655 36027 : unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1656 :
1657 : /* Set COUNT to the upper limit on the number of elements - 1.
1658 : Start with the maximum vectorization factor. */
1659 36027 : unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1660 :
1661 : /* Try lowering COUNT to the number of scalar latch iterations. Peeling
1662 : for alignment by masking shifts the active lanes up, and we cannot tell
1663 : yet whether peeling will be done by masking, or whether masks will be
1664 : needed at all, so be conservative and do not lower COUNT when peeling
1665 : for alignment. */
1666 36027 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1667 36027 : widest_int max_iters;
1668 36027 : if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
1669 36004 : && max_loop_iterations (loop, &max_iters)
1670 71257 : && max_iters < count)
1671 2146 : count = max_iters.to_shwi ();
1672 :
1673 : /* Try scales of 1 and the element size. */
1674 36027 : unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
1675 36027 : wi::overflow_type overflow = wi::OVF_NONE;
1676 108081 : for (int i = 0; i < 2; ++i)
1677 : {
1678 72054 : unsigned int scale = scales[i];
1679 72054 : widest_int factor;
1680 72054 : if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1681 0 : continue;
1682 :
1683 : /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1684 72054 : widest_int range = wi::mul (count, factor, SIGNED, &overflow);
1685 72054 : if (overflow)
1686 0 : continue;
1687 72054 : signop sign = range >= 0 ? UNSIGNED : SIGNED;
1688 72054 : unsigned int min_offset_bits = wi::min_precision (range, sign);
1689 :
1690 : /* Find the narrowest viable offset type. */
1691 72054 : unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
1692 72054 : tree offset_type = build_nonstandard_integer_type (offset_bits,
1693 : sign == UNSIGNED);
1694 :
1695 : /* See whether the target supports the operation with an offset
1696 : no narrower than OFFSET_TYPE. */
1697 72054 : tree memory_type = TREE_TYPE (DR_REF (dr));
1698 72054 : tree tmp_offset_vectype;
1699 72054 : int tmp_scale;
1700 72054 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
1701 : vectype, memory_type, offset_type,
1702 : scale, &tmp_scale,
1703 : &gs_info->ifn, &gs_info->offset_vectype,
1704 : &tmp_offset_vectype, elsvals)
1705 72054 : || gs_info->ifn == IFN_LAST)
1706 72054 : continue;
1707 :
1708 0 : gs_info->decl = NULL_TREE;
1709 : /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1710 : but we don't need to store that here. */
1711 0 : gs_info->base = NULL_TREE;
1712 0 : gs_info->alias_ptr = build_int_cst
1713 0 : (reference_alias_ptr_type (DR_REF (dr)),
1714 0 : get_object_alignment (DR_REF (dr)));
1715 0 : gs_info->element_type = TREE_TYPE (vectype);
1716 0 : gs_info->offset = fold_convert (offset_type, step);
1717 0 : gs_info->scale = scale;
1718 0 : gs_info->memory_type = memory_type;
1719 0 : return true;
1720 72054 : }
1721 :
1722 36027 : if (overflow && dump_enabled_p ())
1723 0 : dump_printf_loc (MSG_NOTE, vect_location,
1724 : "truncating gather/scatter offset to %d bits"
1725 : " might change its value.\n", element_bits);
1726 :
1727 : return false;
1728 65671 : }
1729 :
1730 : /* Return true if we can use gather/scatter or strided internal functions
1731 : to vectorize STMT_INFO, which is a grouped or strided load or store
1732 : with multiple lanes and will be implemented by a type-punned access
1733 : of a vector with element size that matches the number of lanes.
1734 :
1735 : MASKED_P is true if load or store is conditional.
1736 : When returning true, fill in GS_INFO with the information required to
1737 : perform the operation. Also, store the punning type in PUNNED_VECTYPE.
1738 :
1739 : If successful and ELSVALS is nonzero the supported
1740 : else values will be stored in the vector ELSVALS points to. */
1741 :
1742 : static bool
1743 4658 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
1744 : loop_vec_info loop_vinfo, bool masked_p,
1745 : unsigned int nelts,
1746 : gather_scatter_info *info, vec<int> *elsvals,
1747 : tree *pun_vectype)
1748 : {
1749 4658 : data_reference *dr = dr_info->dr;
1750 :
1751 : /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
1752 : multiple gathers/scatter. */
1753 4658 : if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
1754 : return false;
1755 :
1756 : /* Pun the vectype with one of the same size but an element spanning
1757 : NELTS elements of VECTYPE.
1758 : The punned type of a V16QI with NELTS = 4 would be V4SI.
1759 : */
1760 4017 : tree tmp;
1761 4017 : unsigned int pieces;
1762 4017 : if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
1763 4017 : || pieces <= 1)
1764 2003 : return false;
1765 :
1766 2014 : *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
1767 :
1768 2014 : if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
1769 : return false;
1770 :
1771 1868 : internal_fn ifn;
1772 1868 : tree offset_vectype = *pun_vectype;
1773 :
1774 1261 : internal_fn strided_ifn = DR_IS_READ (dr)
1775 1868 : ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
1776 :
1777 : /* Check if we have a gather/scatter with the new type. We're just trying
1778 : with the type itself as offset for now. If not, check if we have a
1779 : strided load/store. These have fewer constraints (for example no offset
1780 : type must exist) so it is possible that even though a gather/scatter is
1781 : not available we still have a strided load/store. */
1782 1868 : bool ok = false;
1783 1868 : tree tmp_vectype;
1784 1868 : int tmp_scale;
1785 1868 : if (vect_gather_scatter_fn_p
1786 1868 : (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
1787 1868 : TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
1788 : &offset_vectype, &tmp_vectype, elsvals))
1789 : ok = true;
1790 1868 : else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
1791 : elsvals))
1792 : {
1793 : /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
1794 : will switch back to the strided variants. */
1795 0 : ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
1796 : IFN_MASK_LEN_SCATTER_STORE;
1797 0 : ok = true;
1798 : }
1799 :
1800 0 : if (ok)
1801 : {
1802 0 : info->ifn = ifn;
1803 0 : info->decl = NULL_TREE;
1804 0 : info->base = dr->ref;
1805 0 : info->alias_ptr = build_int_cst
1806 0 : (reference_alias_ptr_type (DR_REF (dr)),
1807 0 : get_object_alignment (DR_REF (dr)));
1808 0 : info->element_type = TREE_TYPE (*pun_vectype);
1809 0 : info->offset_vectype = offset_vectype;
1810 : /* No need to set the offset, vect_get_strided_load_store_ops
1811 : will do that. */
1812 0 : info->scale = 1;
1813 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
1814 0 : return true;
1815 : }
1816 :
1817 : return false;
1818 : }
1819 :
1820 :
1821 : /* Return true if we can use gather/scatter internal functions to
1822 : vectorize STMT_INFO, which is a grouped or strided load or store.
1823 : MASKED_P is true if load or store is conditional. When returning
1824 : true, fill in GS_INFO with the information required to perform the
1825 : operation.
1826 :
1827 : If we can use gather/scatter and ELSVALS is nonzero the supported
1828 : else values will be stored in the vector ELSVALS points to. */
1829 :
1830 : static bool
1831 65671 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
1832 : loop_vec_info loop_vinfo, bool masked_p,
1833 : gather_scatter_info *gs_info,
1834 : vec<int> *elsvals,
1835 : unsigned int group_size,
1836 : bool single_element_p)
1837 : {
1838 65671 : if (!vect_check_gather_scatter (stmt_info, vectype,
1839 : loop_vinfo, gs_info, elsvals)
1840 65671 : || gs_info->ifn == IFN_LAST)
1841 : {
1842 65671 : if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
1843 : masked_p, gs_info, elsvals))
1844 : return false;
1845 : }
1846 :
1847 0 : if (!single_element_p
1848 0 : && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
1849 : gs_info->scale,
1850 : group_size))
1851 : return false;
1852 :
1853 0 : if (dump_enabled_p ())
1854 0 : dump_printf_loc (MSG_NOTE, vect_location,
1855 : "using gather/scatter for strided/grouped access,"
1856 : " scale = %d\n", gs_info->scale);
1857 :
1858 : return true;
1859 : }
1860 :
1861 : /* STMT_INFO is a non-strided load or store, meaning that it accesses
1862 : elements with a known constant step. Return -1 if that step
1863 : is negative, 0 if it is zero, and 1 if it is greater than zero. */
1864 :
1865 : int
1866 1544118 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
1867 : {
1868 1544118 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1869 1544118 : return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
1870 1544118 : size_zero_node);
1871 : }
1872 :
1873 : /* If the target supports a permute mask that reverses the elements in
1874 : a vector of type VECTYPE, return that mask, otherwise return null. */
1875 :
1876 : tree
1877 9452 : perm_mask_for_reverse (tree vectype)
1878 : {
1879 9452 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1880 :
1881 : /* The encoding has a single stepped pattern. */
1882 9452 : vec_perm_builder sel (nunits, 1, 3);
1883 47260 : for (int i = 0; i < 3; ++i)
1884 28356 : sel.quick_push (nunits - 1 - i);
1885 :
1886 9452 : vec_perm_indices indices (sel, 1, nunits);
1887 9452 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
1888 : indices))
1889 : return NULL_TREE;
1890 8224 : return vect_gen_perm_mask_checked (vectype, indices);
1891 9452 : }
1892 :
1893 : /* A subroutine of get_load_store_type, with a subset of the same
1894 : arguments. Handle the case where STMT_INFO is a load or store that
1895 : accesses consecutive elements with a negative step. Sets *POFFSET
1896 : to the offset to be applied to the DR for the first access. */
1897 :
1898 : static vect_memory_access_type
1899 12434 : get_negative_load_store_type (vec_info *vinfo,
1900 : stmt_vec_info stmt_info, tree vectype,
1901 : vec_load_store_type vls_type,
1902 : unsigned int ncopies, poly_int64 *poffset)
1903 : {
1904 12434 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1905 12434 : dr_alignment_support alignment_support_scheme;
1906 :
1907 12434 : if (ncopies > 1)
1908 : {
1909 0 : if (dump_enabled_p ())
1910 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1911 : "multiple types with negative step.\n");
1912 : return VMAT_ELEMENTWISE;
1913 : }
1914 :
1915 : /* For backward running DRs the first access in vectype actually is
1916 : N-1 elements before the address of the DR. */
1917 12434 : *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
1918 12434 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1919 :
1920 12434 : int misalignment = dr_misalignment (dr_info, vectype, *poffset);
1921 12434 : alignment_support_scheme
1922 12434 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
1923 12434 : if (alignment_support_scheme != dr_aligned
1924 12434 : && alignment_support_scheme != dr_unaligned_supported)
1925 : {
1926 4436 : if (dump_enabled_p ())
1927 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1928 : "negative step but alignment required.\n");
1929 4436 : *poffset = 0;
1930 4436 : return VMAT_ELEMENTWISE;
1931 : }
1932 :
1933 7998 : if (vls_type == VLS_STORE_INVARIANT)
1934 : {
1935 1184 : if (dump_enabled_p ())
1936 21 : dump_printf_loc (MSG_NOTE, vect_location,
1937 : "negative step with invariant source;"
1938 : " no permute needed.\n");
1939 : return VMAT_CONTIGUOUS_DOWN;
1940 : }
1941 :
1942 6814 : if (!perm_mask_for_reverse (vectype))
1943 : {
1944 1228 : if (dump_enabled_p ())
1945 52 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1946 : "negative step and reversing not supported.\n");
1947 1228 : *poffset = 0;
1948 1228 : return VMAT_ELEMENTWISE;
1949 : }
1950 :
1951 : return VMAT_CONTIGUOUS_REVERSE;
1952 : }
1953 :
1954 : /* STMT_INFO is either a masked or unconditional store. Return the value
1955 : being stored. */
1956 :
1957 : tree
1958 0 : vect_get_store_rhs (stmt_vec_info stmt_info)
1959 : {
1960 0 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
1961 : {
1962 0 : gcc_assert (gimple_assign_single_p (assign));
1963 0 : return gimple_assign_rhs1 (assign);
1964 : }
1965 0 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
1966 : {
1967 0 : internal_fn ifn = gimple_call_internal_fn (call);
1968 0 : int index = internal_fn_stored_value_index (ifn);
1969 0 : gcc_assert (index >= 0);
1970 0 : return gimple_call_arg (call, index);
1971 : }
1972 0 : gcc_unreachable ();
1973 : }
1974 :
1975 : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1976 :
1977 : This function returns a vector type which can be composed with NELTS pieces,
1978 : whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1979 : same vector size as the return vector. It checks target whether supports
1980 : pieces-size vector mode for construction firstly, if target fails to, check
1981 : pieces-size scalar mode for construction further. It returns NULL_TREE if
1982 : fails to find the available composition. If the caller only wants scalar
1983 : pieces where PTYPE e.g. is a possible gather/scatter element type
1984 : SCALAR_PTYPE_ONLY must be true.
1985 :
1986 : For example, for (vtype=V16QI, nelts=4), we can probably get:
1987 : - V16QI with PTYPE V4QI.
1988 : - V4SI with PTYPE SI.
1989 : - NULL_TREE. */
1990 :
1991 : static tree
1992 12838 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
1993 : bool scalar_ptype_only)
1994 : {
1995 12838 : gcc_assert (VECTOR_TYPE_P (vtype));
1996 12838 : gcc_assert (known_gt (nelts, 0U));
1997 :
1998 12838 : machine_mode vmode = TYPE_MODE (vtype);
1999 12838 : if (!VECTOR_MODE_P (vmode))
2000 : return NULL_TREE;
2001 :
2002 : /* When we are asked to compose the vector from its components let
2003 : that happen directly. */
2004 12838 : if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
2005 : {
2006 6095 : *ptype = TREE_TYPE (vtype);
2007 6095 : return vtype;
2008 : }
2009 :
2010 13486 : poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
2011 6743 : unsigned int pbsize;
2012 6743 : if (constant_multiple_p (vbsize, nelts, &pbsize))
2013 : {
2014 : /* First check if vec_init optab supports construction from
2015 : vector pieces directly. */
2016 6743 : scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
2017 13486 : poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
2018 6743 : machine_mode rmode;
2019 6743 : if (!scalar_ptype_only
2020 8350 : && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
2021 11010 : && (convert_optab_handler (vec_init_optab, vmode, rmode)
2022 : != CODE_FOR_nothing))
2023 : {
2024 3621 : *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
2025 3621 : return vtype;
2026 : }
2027 :
2028 : /* Otherwise check if exists an integer type of the same piece size and
2029 : if vec_init optab supports construction from it directly. */
2030 3122 : if (int_mode_for_size (pbsize, 0).exists (&elmode)
2031 3122 : && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
2032 : {
2033 2938 : if (scalar_ptype_only
2034 2938 : || convert_optab_handler (vec_init_optab, rmode, elmode)
2035 : != CODE_FOR_nothing)
2036 : {
2037 2938 : *ptype = build_nonstandard_integer_type (pbsize, 1);
2038 2938 : return build_vector_type (*ptype, nelts);
2039 : }
2040 : }
2041 : }
2042 :
2043 : return NULL_TREE;
2044 : }
2045 :
2046 : /* Check if the load permutation of NODE only refers to a consecutive
2047 : subset of the group indices. We also require the length of the
2048 : permutation to be a power of two.
2049 : Such load permutations can be elided in strided access schemes as
2050 : we can "jump over" the gap they leave. */
2051 :
2052 : static bool
2053 46339 : has_consecutive_load_permutation (slp_tree node)
2054 : {
2055 46339 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
2056 46339 : if (!perm.exists ()
2057 2183 : || perm.length () <= 1
2058 46848 : || !pow2p_hwi (perm.length ()))
2059 : return false;
2060 :
2061 489 : return vect_load_perm_consecutive_p (node);
2062 : }
2063 :
2064 :
2065 : /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
2066 : if there is a memory access type that the vectorized form can use,
2067 : storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2068 : or scatters, fill in GS_INFO accordingly. In addition
2069 : *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2070 : the target does not support the alignment scheme. *MISALIGNMENT
2071 : is set according to the alignment of the access (including
2072 : DR_MISALIGNMENT_UNKNOWN when it is unknown).
2073 :
2074 : MASKED_P is true if the statement is conditional on a vectorized mask.
2075 : VECTYPE is the vector type that the vectorized statements will use.
2076 :
2077 : If ELSVALS is nonzero the supported else values will be stored in the
2078 : vector ELSVALS points to. */
2079 :
2080 : static bool
2081 1421714 : get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
2082 : tree vectype, slp_tree slp_node,
2083 : bool masked_p, vec_load_store_type vls_type,
2084 : vect_load_store_data *ls)
2085 : {
2086 1421714 : vect_memory_access_type *memory_access_type = &ls->memory_access_type;
2087 1421714 : poly_int64 *poffset = &ls->poffset;
2088 1421714 : dr_alignment_support *alignment_support_scheme
2089 : = &ls->alignment_support_scheme;
2090 1421714 : int *misalignment = &ls->misalignment;
2091 1421714 : internal_fn *lanes_ifn = &ls->lanes_ifn;
2092 1421714 : vec<int> *elsvals = &ls->elsvals;
2093 1421714 : tree *ls_type = &ls->ls_type;
2094 1421714 : bool *slp_perm = &ls->slp_perm;
2095 1421714 : unsigned *n_perms = &ls->n_perms;
2096 1421714 : unsigned *n_loads = &ls->n_loads;
2097 1421714 : tree *supported_offset_vectype = &ls->supported_offset_vectype;
2098 1421714 : int *supported_scale = &ls->supported_scale;
2099 1421714 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
2100 1421714 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2101 1421714 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2102 1421714 : stmt_vec_info first_stmt_info;
2103 1421714 : unsigned int group_size;
2104 1421714 : unsigned HOST_WIDE_INT gap;
2105 1421714 : bool single_element_p;
2106 1421714 : poly_int64 neg_ldst_offset = 0;
2107 :
2108 1421714 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2109 1421714 : *poffset = 0;
2110 1421714 : *ls_type = NULL_TREE;
2111 1421714 : *slp_perm = false;
2112 1421714 : *n_perms = -1U;
2113 1421714 : *n_loads = -1U;
2114 1421714 : ls->subchain_p = false;
2115 :
2116 1421714 : bool perm_ok = true;
2117 1421714 : poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
2118 :
2119 1421714 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2120 78907 : perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
2121 78907 : vf, true, n_perms, n_loads);
2122 :
2123 1421714 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2124 : {
2125 899684 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
2126 899684 : group_size = DR_GROUP_SIZE (first_stmt_info);
2127 899684 : gap = DR_GROUP_GAP (first_stmt_info);
2128 899684 : single_element_p = (stmt_info == first_stmt_info
2129 899684 : && !DR_GROUP_NEXT_ELEMENT (stmt_info));
2130 : }
2131 : else
2132 : {
2133 : first_stmt_info = stmt_info;
2134 : group_size = 1;
2135 : gap = 0;
2136 : single_element_p = true;
2137 : }
2138 1421714 : dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
2139 :
2140 : /* There can only be a gap at the end of the group if the stride is
2141 : known at compile time. */
2142 1421714 : gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
2143 :
2144 : /* For SLP vectorization we directly vectorize a subchain
2145 : without permutation. */
2146 1421714 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2147 1342807 : first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
2148 :
2149 1421714 : if (STMT_VINFO_STRIDED_P (first_stmt_info))
2150 : {
2151 : /* Try to use consecutive accesses of as many elements as possible,
2152 : separated by the stride, until we have a complete vector.
2153 : Fall back to scalar accesses if that isn't possible. */
2154 46339 : *memory_access_type = VMAT_STRIDED_SLP;
2155 :
2156 : /* If the load permutation is consecutive we can reduce the group to
2157 : the elements the permutation accesses. Then we release the
2158 : permutation. */
2159 46339 : if (has_consecutive_load_permutation (slp_node))
2160 : {
2161 51 : ls->subchain_p = true;
2162 51 : group_size = SLP_TREE_LANES (slp_node);
2163 51 : SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
2164 : }
2165 : }
2166 1375375 : else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2167 : {
2168 11249 : slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
2169 11249 : tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
2170 11249 : int scale = SLP_TREE_GS_SCALE (slp_node);
2171 11249 : tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
2172 11249 : tree tem;
2173 11249 : if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
2174 : masked_p, vectype, memory_type,
2175 : offset_vectype, scale, supported_scale,
2176 : &ls->gs.ifn, &tem,
2177 : supported_offset_vectype, elsvals))
2178 : {
2179 0 : if (dump_enabled_p ())
2180 : {
2181 0 : dump_printf_loc (MSG_NOTE, vect_location,
2182 : "gather/scatter with required "
2183 : "offset type "
2184 : "%T and offset scale %d.\n",
2185 : offset_vectype, scale);
2186 0 : if (*supported_offset_vectype)
2187 0 : dump_printf_loc (MSG_NOTE, vect_location,
2188 : " target supports offset type %T.\n",
2189 : *supported_offset_vectype);
2190 0 : if (*supported_scale)
2191 0 : dump_printf_loc (MSG_NOTE, vect_location,
2192 : " target supports offset scale %d.\n",
2193 : *supported_scale);
2194 : }
2195 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2196 : }
2197 11249 : else if (vls_type == VLS_LOAD
2198 11249 : ? (targetm.vectorize.builtin_gather
2199 9842 : && (ls->gs.decl
2200 9842 : = targetm.vectorize.builtin_gather (vectype,
2201 9842 : TREE_TYPE
2202 : (offset_vectype),
2203 : scale)))
2204 1407 : : (targetm.vectorize.builtin_scatter
2205 1407 : && (ls->gs.decl
2206 1407 : = targetm.vectorize.builtin_scatter (vectype,
2207 1407 : TREE_TYPE
2208 : (offset_vectype),
2209 : scale))))
2210 569 : *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
2211 : else
2212 : {
2213 : /* GATHER_SCATTER_EMULATED_P. */
2214 10680 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
2215 10680 : || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
2216 10680 : || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
2217 10680 : || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
2218 14158 : TYPE_VECTOR_SUBPARTS (vectype)))
2219 : {
2220 3478 : if (dump_enabled_p ())
2221 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2222 : "unsupported vector types for emulated "
2223 : "gather.\n");
2224 3478 : return false;
2225 : }
2226 7202 : *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
2227 : }
2228 : }
2229 : else
2230 : {
2231 1364126 : int cmp = compare_step_with_zero (vinfo, stmt_info);
2232 1364126 : if (cmp < 0)
2233 : {
2234 12618 : if (single_element_p)
2235 : /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2236 : only correct for single element "interleaving" SLP. */
2237 12434 : *memory_access_type = get_negative_load_store_type
2238 12434 : (vinfo, stmt_info, vectype, vls_type, 1,
2239 : &neg_ldst_offset);
2240 : else
2241 : /* We can fall back to VMAT_STRIDED_SLP since that does
2242 : not care whether the stride between the group instances
2243 : is positive or negative. */
2244 184 : *memory_access_type = VMAT_STRIDED_SLP;
2245 : }
2246 1351508 : else if (cmp == 0 && loop_vinfo)
2247 : {
2248 5144 : gcc_assert (vls_type == VLS_LOAD);
2249 5144 : *memory_access_type = VMAT_INVARIANT;
2250 : }
2251 : /* Try using LOAD/STORE_LANES. */
2252 1346364 : else if (slp_node->ldst_lanes
2253 1346364 : && (*lanes_ifn
2254 0 : = (vls_type == VLS_LOAD
2255 0 : ? vect_load_lanes_supported (vectype, group_size,
2256 : masked_p, elsvals)
2257 0 : : vect_store_lanes_supported (vectype, group_size,
2258 : masked_p))) != IFN_LAST)
2259 0 : *memory_access_type = VMAT_LOAD_STORE_LANES;
2260 1346364 : else if (!loop_vinfo && slp_node->avoid_stlf_fail)
2261 : {
2262 66 : *memory_access_type = VMAT_ELEMENTWISE;
2263 66 : if (dump_enabled_p ())
2264 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2265 : "using element-wise load to avoid disrupting "
2266 : "cross iteration store-to-load forwarding\n");
2267 : }
2268 : else
2269 1346298 : *memory_access_type = VMAT_CONTIGUOUS;
2270 :
2271 : /* True if the vectorized statements would access beyond the last
2272 : statement in the group. */
2273 1364126 : bool overrun_p = false;
2274 :
2275 : /* True if we can cope with such overrun by peeling for gaps, so that
2276 : there is at least one final scalar iteration after the vector loop. */
2277 2728252 : bool can_overrun_p = (!masked_p
2278 1364126 : && vls_type == VLS_LOAD
2279 552738 : && loop_vinfo
2280 408811 : && !loop->inner
2281 1771085 : && (*memory_access_type != VMAT_STRIDED_SLP
2282 80 : || cmp > 0));
2283 :
2284 : /* If this is single-element interleaving with an element
2285 : distance that leaves unused vector loads around fall back
2286 : to elementwise access if possible - we otherwise least
2287 : create very sub-optimal code in that case (and
2288 : blow up memory, see PR65518). */
2289 1364126 : if (loop_vinfo
2290 1364126 : && single_element_p
2291 501783 : && (*memory_access_type == VMAT_CONTIGUOUS
2292 17578 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2293 1865909 : && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
2294 : {
2295 18285 : *memory_access_type = VMAT_ELEMENTWISE;
2296 18285 : if (dump_enabled_p ())
2297 198 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2298 : "single-element interleaving not supported "
2299 : "for not adjacent vector loads, using "
2300 : "elementwise access\n");
2301 : }
2302 :
2303 : /* Also fall back to elementwise access in case we did not lower a
2304 : permutation and cannot code generate it. */
2305 1364126 : if (loop_vinfo
2306 552884 : && *memory_access_type != VMAT_ELEMENTWISE
2307 528935 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2308 1390063 : && !perm_ok)
2309 : {
2310 2106 : *memory_access_type = VMAT_ELEMENTWISE;
2311 2106 : if (dump_enabled_p ())
2312 248 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2313 : "permutation not supported, using elementwise "
2314 : "access\n");
2315 : }
2316 :
2317 552884 : overrun_p = (loop_vinfo && gap != 0
2318 1407911 : && *memory_access_type != VMAT_ELEMENTWISE);
2319 1364126 : if (overrun_p && vls_type != VLS_LOAD)
2320 : {
2321 0 : if (dump_enabled_p ())
2322 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2323 : "Grouped store with gaps requires"
2324 : " non-consecutive accesses\n");
2325 23 : return false;
2326 : }
2327 :
2328 1364126 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
2329 1364126 : poly_int64 off = 0;
2330 1364126 : if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2331 5427 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2332 :
2333 : /* An overrun is fine if the trailing elements are smaller
2334 : than the alignment boundary B. Every vector access will
2335 : be a multiple of B and so we are guaranteed to access a
2336 : non-gap element in the same B-sized block. */
2337 1364126 : if (overrun_p
2338 1364126 : && gap < (vect_known_alignment_in_bytes (first_dr_info,
2339 23325 : vectype, off) / dr_size))
2340 : overrun_p = false;
2341 :
2342 : /* When we have a contiguous access across loop iterations
2343 : but the access in the loop doesn't cover the full vector
2344 : we can end up with no gap recorded but still excess
2345 : elements accessed, see PR103116. Make sure we peel for
2346 : gaps if necessary and sufficient and give up if not.
2347 :
2348 : If there is a combination of the access not covering the full
2349 : vector and a gap recorded then we may need to peel twice. */
2350 1364126 : bool large_vector_overrun_p = false;
2351 1364126 : if (loop_vinfo
2352 552884 : && (*memory_access_type == VMAT_CONTIGUOUS
2353 37989 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2354 520322 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2355 1387573 : && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
2356 : nunits))
2357 : large_vector_overrun_p = overrun_p = true;
2358 :
2359 : /* If the gap splits the vector in half and the target
2360 : can do half-vector operations avoid the epilogue peeling
2361 : by simply loading half of the vector only. Usually
2362 : the construction with an upper zero half will be elided. */
2363 1364126 : dr_alignment_support alss;
2364 1364126 : int misalign = dr_misalignment (first_dr_info, vectype, off);
2365 1364126 : tree half_vtype;
2366 1364126 : poly_uint64 remain;
2367 1364126 : unsigned HOST_WIDE_INT tem, num;
2368 1364126 : if (overrun_p
2369 1364126 : && !masked_p
2370 17795 : && (*memory_access_type == VMAT_CONTIGUOUS
2371 360 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2372 17781 : && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
2373 : vectype, misalign)))
2374 : == dr_aligned
2375 15246 : || alss == dr_unaligned_supported)
2376 10028 : && can_div_trunc_p (group_size
2377 10028 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2378 : nunits, &tem, &remain)
2379 1374154 : && (known_eq (remain, 0u)
2380 7525 : || (known_ne (remain, 0u)
2381 5825 : && constant_multiple_p (nunits, remain, &num)
2382 1364126 : && (vector_vector_composition_type (vectype, num, &half_vtype)
2383 : != NULL_TREE))))
2384 : overrun_p = false;
2385 :
2386 1364126 : if (overrun_p && !can_overrun_p)
2387 : {
2388 20 : if (dump_enabled_p ())
2389 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2390 : "Peeling for outer loop is not supported\n");
2391 : return false;
2392 : }
2393 :
2394 : /* Peeling for gaps assumes that a single scalar iteration
2395 : is enough to make sure the last vector iteration doesn't
2396 : access excess elements. */
2397 1364106 : if (overrun_p
2398 1364106 : && (!can_div_trunc_p (group_size
2399 9447 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2400 : nunits, &tem, &remain)
2401 9447 : || maybe_lt (remain + group_size, nunits)))
2402 : {
2403 : /* But peeling a single scalar iteration is enough if
2404 : we can use the next power-of-two sized partial
2405 : access and that is sufficiently small to be covered
2406 : by the single scalar iteration. */
2407 16 : unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
2408 16 : if (masked_p
2409 16 : || !nunits.is_constant (&cnunits)
2410 16 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
2411 16 : || (((cremain = (group_size * cvf - gap) % cnunits), true)
2412 16 : && ((cpart_size = (1 << ceil_log2 (cremain))), true)
2413 16 : && (cremain + group_size < cpart_size
2414 16 : || (vector_vector_composition_type (vectype,
2415 13 : cnunits / cpart_size,
2416 : &half_vtype)
2417 : == NULL_TREE))))
2418 : {
2419 : /* If all fails we can still resort to niter masking unless
2420 : the vectors used are too big, so enforce the use of
2421 : partial vectors. */
2422 3 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2423 3 : && !large_vector_overrun_p)
2424 : {
2425 0 : if (dump_enabled_p ())
2426 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2427 : "peeling for gaps insufficient for "
2428 : "access unless using partial "
2429 : "vectors\n");
2430 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2431 : }
2432 : else
2433 : {
2434 3 : if (dump_enabled_p ())
2435 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2436 : "peeling for gaps insufficient for "
2437 : "access\n");
2438 23 : return false;
2439 : }
2440 : }
2441 13 : else if (large_vector_overrun_p)
2442 : {
2443 13 : if (dump_enabled_p ())
2444 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2445 : "can't operate on partial vectors because "
2446 : "only unmasked loads handle access "
2447 : "shortening required because of gaps at "
2448 : "the end of the access\n");
2449 13 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2450 : }
2451 : }
2452 :
2453 1364103 : if (overrun_p)
2454 : {
2455 9444 : gcc_assert (can_overrun_p);
2456 9444 : if (dump_enabled_p ())
2457 511 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2458 : "Data access with gaps requires scalar "
2459 : "epilogue loop\n");
2460 9444 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2461 : }
2462 : }
2463 :
2464 : /* As a last resort, trying using a gather load or scatter store.
2465 :
2466 : ??? Although the code can handle all group sizes correctly,
2467 : it probably isn't a win to use separate strided accesses based
2468 : on nearby locations. Or, even if it's a win over scalar code,
2469 : it might not be a win over vectorizing at a lower VF, if that
2470 : allows us to use contiguous accesses. */
2471 1418213 : vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
2472 1418213 : if (loop_vinfo
2473 606971 : && (*memory_access_type == VMAT_ELEMENTWISE
2474 606971 : || *memory_access_type == VMAT_STRIDED_SLP))
2475 : {
2476 72559 : gather_scatter_info gs_info;
2477 72559 : tree tem;
2478 72559 : if (SLP_TREE_LANES (slp_node) == 1
2479 67707 : && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2480 22099 : || single_element_p)
2481 138230 : && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
2482 : masked_p, &gs_info, elsvals,
2483 : group_size, single_element_p))
2484 : {
2485 : /* vect_use_strided_gather_scatters_p does not save the actually
2486 : supported scale and offset type so do that here.
2487 : We need it later in check_load_store_for_partial_vectors
2488 : where we only check if the given internal function is supported
2489 : (to choose whether to use the IFN, LEGACY, or EMULATED flavor
2490 : of gather/scatter) and don't re-do the full analysis. */
2491 0 : tree tmp;
2492 0 : gcc_assert (vect_gather_scatter_fn_p
2493 : (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
2494 : gs_info.memory_type, TREE_TYPE (gs_info.offset),
2495 : gs_info.scale, supported_scale, &gs_info.ifn,
2496 : &tmp, supported_offset_vectype, elsvals));
2497 :
2498 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2499 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2500 0 : ls->gs.ifn = gs_info.ifn;
2501 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2502 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2503 : }
2504 72559 : else if (SLP_TREE_LANES (slp_node) > 1
2505 : && !masked_p
2506 4852 : && !single_element_p
2507 77217 : && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
2508 : vectype, loop_vinfo,
2509 : masked_p, group_size,
2510 : &gs_info, elsvals, &tem))
2511 : {
2512 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2513 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2514 0 : grouped_gather_fallback = *memory_access_type;
2515 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2516 0 : ls->gs.ifn = gs_info.ifn;
2517 0 : vectype = *ls_type = tem;
2518 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2519 : }
2520 : }
2521 :
2522 1418213 : if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
2523 1418213 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2524 6607 : *poffset = neg_ldst_offset;
2525 :
2526 1418213 : if (*memory_access_type == VMAT_ELEMENTWISE
2527 1392092 : || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
2528 1391523 : || *memory_access_type == VMAT_STRIDED_SLP
2529 1345019 : || *memory_access_type == VMAT_INVARIANT)
2530 : {
2531 78338 : *alignment_support_scheme = dr_unaligned_supported;
2532 78338 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2533 : }
2534 : else
2535 : {
2536 1339875 : if (mat_gather_scatter_p (*memory_access_type)
2537 : && !first_dr_info)
2538 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2539 : else
2540 1339875 : *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
2541 1339875 : *alignment_support_scheme
2542 1339875 : = vect_supportable_dr_alignment
2543 1339875 : (vinfo, first_dr_info, vectype, *misalignment,
2544 : mat_gather_scatter_p (*memory_access_type));
2545 1339875 : if (grouped_gather_fallback != VMAT_UNINITIALIZED
2546 0 : && *alignment_support_scheme != dr_aligned
2547 0 : && *alignment_support_scheme != dr_unaligned_supported)
2548 : {
2549 : /* No supportable alignment for a grouped gather, fall back to the
2550 : original memory access type. Even though VMAT_STRIDED_SLP might
2551 : also try aligned vector loads it can still choose vector
2552 : construction from scalars. */
2553 0 : *memory_access_type = grouped_gather_fallback;
2554 0 : *alignment_support_scheme = dr_unaligned_supported;
2555 0 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2556 : }
2557 : }
2558 :
2559 1418213 : if ((*memory_access_type == VMAT_ELEMENTWISE
2560 1418213 : || *memory_access_type == VMAT_STRIDED_SLP)
2561 : && !nunits.is_constant ())
2562 : {
2563 : if (dump_enabled_p ())
2564 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2565 : "Not using elementwise accesses due to variable "
2566 : "vectorization factor.\n");
2567 : return false;
2568 : }
2569 :
2570 : /* Checks if all scalar iterations are known to be inbounds. */
2571 1418213 : bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
2572 :
2573 : /* Check if we support the operation if early breaks are needed. Here we
2574 : must ensure that we don't access any more than the scalar code would
2575 : have. A masked operation would ensure this, so for these load types
2576 : force masking. */
2577 1418213 : if (loop_vinfo
2578 606971 : && dr_safe_speculative_read_required (stmt_info)
2579 1618911 : && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2580 : {
2581 200698 : if (mat_gather_scatter_p (*memory_access_type)
2582 200698 : || *memory_access_type == VMAT_STRIDED_SLP)
2583 : {
2584 9542 : if (dump_enabled_p ())
2585 8 : dump_printf_loc (MSG_NOTE, vect_location,
2586 : "early break not supported: cannot peel for "
2587 : "alignment. With non-contiguous memory vectorization"
2588 : " could read out of bounds at %G ",
2589 : STMT_VINFO_STMT (stmt_info));
2590 9542 : if (inbounds)
2591 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2592 : else
2593 : return false;
2594 : }
2595 : /* Block-level alignment: Even though individual accesses of
2596 : VMAT_ELEMENTWISE type do not cause alignment problems, loading the
2597 : whole vector's worth of values in a speculative early-break context
2598 : might cross a page boundary. Set the alignment scheme to `dr_aligned'
2599 : here in order to force checking of whether such accesses meet
2600 : alignment criteria. */
2601 191156 : else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
2602 15078 : *alignment_support_scheme = dr_aligned;
2603 : }
2604 :
2605 : /* If this DR needs alignment for correctness, we must ensure the target
2606 : alignment is a constant power-of-two multiple of the amount read per
2607 : vector iteration or force masking. */
2608 1408671 : if (dr_safe_speculative_read_required (stmt_info)
2609 1408671 : && (*alignment_support_scheme == dr_aligned
2610 112902 : && !mat_gather_scatter_p (*memory_access_type)))
2611 : {
2612 : /* We can only peel for loops, of course. */
2613 112902 : gcc_checking_assert (loop_vinfo);
2614 :
2615 112902 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2616 112902 : poly_uint64 read_amount
2617 112902 : = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
2618 112902 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2619 112902 : read_amount *= group_size;
2620 :
2621 112902 : auto target_alignment
2622 112902 : = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
2623 112902 : if (!multiple_p (target_alignment, read_amount))
2624 : {
2625 12832 : if (dump_enabled_p ())
2626 : {
2627 28 : dump_printf_loc (MSG_NOTE, vect_location,
2628 : "desired alignment not met, target was ");
2629 28 : dump_dec (MSG_NOTE, target_alignment);
2630 28 : dump_printf (MSG_NOTE, " previously, but read amount is ");
2631 28 : dump_dec (MSG_NOTE, read_amount);
2632 28 : dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
2633 : }
2634 15117 : return false;
2635 : }
2636 :
2637 : /* When using a group access the first element may be aligned but the
2638 : subsequent loads may not be. For LOAD_LANES since the loads are based
2639 : on the first DR then all loads in the group are aligned. For
2640 : non-LOAD_LANES this is not the case. In particular a load + blend when
2641 : there are gaps can have the non first loads issued unaligned, even
2642 : partially overlapping the memory of the first load in order to simplify
2643 : the blend. This is what the x86_64 backend does for instance. As
2644 : such only the first load in the group is aligned, the rest are not.
2645 : Because of this the permutes may break the alignment requirements that
2646 : have been set, and as such we should for now, reject them. */
2647 100070 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2648 : {
2649 2285 : if (dump_enabled_p ())
2650 87 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2651 : "loads with load permutations not supported for "
2652 : "speculative early break loads for %G",
2653 : STMT_VINFO_STMT (stmt_info));
2654 : return false;
2655 : }
2656 :
2657 : /* Reject vectorization if we know the read mount per vector iteration
2658 : exceeds the min page size. */
2659 97785 : if (known_gt (read_amount, (unsigned) param_min_pagesize))
2660 : {
2661 0 : if (dump_enabled_p ())
2662 : {
2663 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2664 : "alignment required for correctness (");
2665 0 : dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
2666 0 : dump_printf (MSG_NOTE, ") may exceed page size.\n");
2667 : }
2668 : return false;
2669 : }
2670 :
2671 97785 : if (!vf.is_constant ())
2672 : {
2673 : /* For VLA modes, we need a runtime check to ensure any speculative
2674 : read amount does not exceed the page size. Here we record the max
2675 : possible read amount for the check. */
2676 : if (maybe_gt (read_amount,
2677 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
2678 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
2679 :
2680 : /* For VLA modes, we must use partial vectors. */
2681 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2682 : }
2683 : }
2684 :
2685 1393554 : if (*alignment_support_scheme == dr_unaligned_unsupported)
2686 : {
2687 72984 : if (dump_enabled_p ())
2688 256 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2689 : "unsupported unaligned access\n");
2690 : return false;
2691 : }
2692 :
2693 : /* FIXME: At the moment the cost model seems to underestimate the
2694 : cost of using elementwise accesses. This check preserves the
2695 : traditional behavior until that can be fixed. */
2696 1320570 : if (*memory_access_type == VMAT_ELEMENTWISE
2697 15260 : && !STMT_VINFO_STRIDED_P (first_stmt_info)
2698 1335830 : && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
2699 10019 : && single_element_p
2700 9376 : && !pow2p_hwi (group_size)))
2701 : {
2702 9337 : if (dump_enabled_p ())
2703 364 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2704 : "not falling back to elementwise accesses\n");
2705 : return false;
2706 : }
2707 :
2708 : /* For BB vectorization build up the vector from existing scalar defs. */
2709 1311233 : if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
2710 : return false;
2711 :
2712 : /* Some loads need to explicitly permute the loaded data if there
2713 : is a load permutation. Among those are:
2714 : - VMAT_ELEMENTWISE.
2715 : - VMAT_STRIDED_SLP.
2716 : - VMAT_GATHER_SCATTER:
2717 : - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
2718 : - Grouped strided gather (ditto but for #lanes > 1).
2719 :
2720 : For VMAT_ELEMENTWISE we can fold the load permutation into the
2721 : individual indices we access directly, eliding the permutation.
2722 : Strided gather only allows load permutations for the
2723 : single-element case. */
2724 :
2725 1311233 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2726 1311233 : && !(*memory_access_type == VMAT_ELEMENTWISE
2727 51238 : || (mat_gather_scatter_p (*memory_access_type)
2728 0 : && SLP_TREE_LANES (slp_node) == 1
2729 0 : && single_element_p)))
2730 : {
2731 51238 : if (!loop_vinfo)
2732 : {
2733 : /* In BB vectorization we may not actually use a loaded vector
2734 : accessing elements in excess of DR_GROUP_SIZE. */
2735 32195 : stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
2736 32195 : group_info = DR_GROUP_FIRST_ELEMENT (group_info);
2737 32195 : unsigned HOST_WIDE_INT nunits;
2738 32195 : unsigned j, k, maxk = 0;
2739 112603 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
2740 80408 : if (k > maxk)
2741 : maxk = k;
2742 32195 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2743 58471 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
2744 32195 : || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
2745 : {
2746 5919 : if (dump_enabled_p ())
2747 41 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2748 : "BB vectorization with gaps at the end of "
2749 : "a load is not supported\n");
2750 1421714 : return false;
2751 : }
2752 : }
2753 :
2754 45319 : if (!perm_ok)
2755 : {
2756 2317 : if (dump_enabled_p ())
2757 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
2758 : vect_location,
2759 : "unsupported load permutation\n");
2760 : return false;
2761 : }
2762 :
2763 43002 : *slp_perm = true;
2764 : }
2765 :
2766 : return true;
2767 : }
2768 :
2769 : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2770 : conditional operation STMT_INFO. When returning true, store the mask
2771 : in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
2772 : the vectorized mask in *MASK_VECTYPE_OUT. */
2773 :
2774 : static bool
2775 12605 : vect_check_scalar_mask (vec_info *vinfo,
2776 : slp_tree slp_node, unsigned mask_index,
2777 : slp_tree *mask_node,
2778 : vect_def_type *mask_dt_out, tree *mask_vectype_out)
2779 : {
2780 12605 : enum vect_def_type mask_dt;
2781 12605 : tree mask_vectype;
2782 12605 : slp_tree mask_node_1;
2783 12605 : tree mask_;
2784 12605 : if (!vect_is_simple_use (vinfo, slp_node, mask_index,
2785 : &mask_, &mask_node_1, &mask_dt, &mask_vectype))
2786 : {
2787 0 : if (dump_enabled_p ())
2788 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2789 : "mask use not simple.\n");
2790 : return false;
2791 : }
2792 :
2793 12605 : if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
2794 12605 : && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
2795 : {
2796 0 : if (dump_enabled_p ())
2797 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2798 : "mask argument is not a boolean.\n");
2799 : return false;
2800 : }
2801 :
2802 12605 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2803 12605 : if (!mask_vectype)
2804 19 : mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
2805 : mask_node_1);
2806 :
2807 12605 : if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2808 : {
2809 0 : if (dump_enabled_p ())
2810 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2811 : "could not find an appropriate vector mask type.\n");
2812 : return false;
2813 : }
2814 :
2815 12605 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2816 25210 : TYPE_VECTOR_SUBPARTS (vectype)))
2817 : {
2818 0 : if (dump_enabled_p ())
2819 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2820 : "vector mask type %T"
2821 : " does not match vector data type %T.\n",
2822 : mask_vectype, vectype);
2823 :
2824 : return false;
2825 : }
2826 :
2827 12605 : *mask_dt_out = mask_dt;
2828 12605 : *mask_vectype_out = mask_vectype;
2829 12605 : *mask_node = mask_node_1;
2830 12605 : return true;
2831 : }
2832 :
2833 :
2834 : /* Return true if stored value is suitable for vectorizing store
2835 : statement STMT_INFO. When returning true, store the scalar stored
2836 : in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2837 : the type of the vectorized store value in
2838 : *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2839 :
2840 : static bool
2841 1387948 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
2842 : slp_tree slp_node, slp_tree *rhs_node,
2843 : vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
2844 : vec_load_store_type *vls_type_out)
2845 : {
2846 1387948 : int op_no = 0;
2847 1387948 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
2848 : {
2849 1883 : if (gimple_call_internal_p (call)
2850 1883 : && internal_store_fn_p (gimple_call_internal_fn (call)))
2851 1883 : op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
2852 : }
2853 1387948 : op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
2854 :
2855 1387948 : enum vect_def_type rhs_dt;
2856 1387948 : tree rhs_vectype;
2857 1387948 : tree rhs;
2858 1387948 : if (!vect_is_simple_use (vinfo, slp_node, op_no,
2859 : &rhs, rhs_node, &rhs_dt, &rhs_vectype))
2860 : {
2861 0 : if (dump_enabled_p ())
2862 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2863 : "use not simple.\n");
2864 : return false;
2865 : }
2866 :
2867 : /* In the case this is a store from a constant make sure
2868 : native_encode_expr can handle it. */
2869 1387948 : if (rhs_dt == vect_constant_def
2870 1387948 : && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2871 : {
2872 0 : if (dump_enabled_p ())
2873 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2874 : "cannot encode constant as a byte sequence.\n");
2875 : return false;
2876 : }
2877 :
2878 1387948 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2879 1387948 : if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2880 : {
2881 24 : if (dump_enabled_p ())
2882 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2883 : "incompatible vector types.\n");
2884 : return false;
2885 : }
2886 :
2887 1387924 : *rhs_dt_out = rhs_dt;
2888 1387924 : *rhs_vectype_out = rhs_vectype;
2889 1387924 : if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2890 1026777 : *vls_type_out = VLS_STORE_INVARIANT;
2891 : else
2892 361147 : *vls_type_out = VLS_STORE;
2893 : return true;
2894 : }
2895 :
2896 : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2897 : Note that we support masks with floating-point type, in which case the
2898 : floats are interpreted as a bitmask. */
2899 :
2900 : static tree
2901 155 : vect_build_all_ones_mask (vec_info *vinfo,
2902 : stmt_vec_info stmt_info, tree masktype)
2903 : {
2904 155 : if (TREE_CODE (masktype) == INTEGER_TYPE)
2905 98 : return build_int_cst (masktype, -1);
2906 57 : else if (VECTOR_BOOLEAN_TYPE_P (masktype)
2907 114 : || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2908 : {
2909 8 : tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2910 8 : mask = build_vector_from_val (masktype, mask);
2911 8 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2912 : }
2913 49 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2914 : {
2915 : REAL_VALUE_TYPE r;
2916 : long tmp[6];
2917 343 : for (int j = 0; j < 6; ++j)
2918 294 : tmp[j] = -1;
2919 49 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2920 49 : tree mask = build_real (TREE_TYPE (masktype), r);
2921 49 : mask = build_vector_from_val (masktype, mask);
2922 49 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2923 : }
2924 0 : gcc_unreachable ();
2925 : }
2926 :
2927 : /* Build an all-zero merge value of type VECTYPE while vectorizing
2928 : STMT_INFO as a gather load. */
2929 :
2930 : static tree
2931 154 : vect_build_zero_merge_argument (vec_info *vinfo,
2932 : stmt_vec_info stmt_info, tree vectype)
2933 : {
2934 154 : tree merge;
2935 154 : if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2936 49 : merge = build_int_cst (TREE_TYPE (vectype), 0);
2937 105 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2938 : {
2939 : REAL_VALUE_TYPE r;
2940 : long tmp[6];
2941 735 : for (int j = 0; j < 6; ++j)
2942 630 : tmp[j] = 0;
2943 105 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2944 105 : merge = build_real (TREE_TYPE (vectype), r);
2945 : }
2946 : else
2947 0 : gcc_unreachable ();
2948 154 : merge = build_vector_from_val (vectype, merge);
2949 154 : return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
2950 : }
2951 :
2952 : /* Return the corresponding else value for an else value constant
2953 : ELSVAL with type TYPE. */
2954 :
2955 : tree
2956 1860 : vect_get_mask_load_else (int elsval, tree type)
2957 : {
2958 1860 : tree els;
2959 1860 : if (elsval == MASK_LOAD_ELSE_UNDEFINED)
2960 : {
2961 0 : tree tmp = create_tmp_var (type);
2962 : /* No need to warn about anything. */
2963 0 : TREE_NO_WARNING (tmp) = 1;
2964 0 : els = get_or_create_ssa_default_def (cfun, tmp);
2965 : }
2966 1860 : else if (elsval == MASK_LOAD_ELSE_M1)
2967 0 : els = build_minus_one_cst (type);
2968 1860 : else if (elsval == MASK_LOAD_ELSE_ZERO)
2969 1860 : els = build_zero_cst (type);
2970 : else
2971 0 : gcc_unreachable ();
2972 :
2973 1860 : return els;
2974 : }
2975 :
2976 : /* Build a gather load call while vectorizing STMT_INFO. Insert new
2977 : instructions before GSI and add them to VEC_STMT. GS_INFO describes
2978 : the gather load operation. If the load is conditional, MASK is the
2979 : vectorized condition, otherwise MASK is null. PTR is the base
2980 : pointer and OFFSET is the vectorized offset. */
2981 :
2982 : static gimple *
2983 337 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
2984 : slp_tree slp_node, tree vectype,
2985 : gimple_stmt_iterator *gsi, tree decl,
2986 : tree ptr, tree offset, tree mask)
2987 : {
2988 337 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
2989 337 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
2990 337 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2991 337 : /* ptrtype */ arglist = TREE_CHAIN (arglist);
2992 337 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2993 337 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2994 337 : tree scaletype = TREE_VALUE (arglist);
2995 337 : tree var;
2996 337 : gcc_checking_assert (types_compatible_p (srctype, rettype)
2997 : && (!mask
2998 : || TREE_CODE (masktype) == INTEGER_TYPE
2999 : || types_compatible_p (srctype, masktype)));
3000 :
3001 337 : tree op = offset;
3002 337 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3003 : {
3004 99 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3005 : TYPE_VECTOR_SUBPARTS (idxtype)));
3006 99 : var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3007 99 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3008 99 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3009 99 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3010 99 : op = var;
3011 : }
3012 :
3013 337 : tree src_op = NULL_TREE;
3014 337 : tree mask_op = NULL_TREE;
3015 337 : if (mask)
3016 : {
3017 183 : if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
3018 : {
3019 183 : tree utype, optype = TREE_TYPE (mask);
3020 183 : if (VECTOR_TYPE_P (masktype)
3021 183 : || TYPE_MODE (masktype) == TYPE_MODE (optype))
3022 : utype = masktype;
3023 : else
3024 6 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3025 183 : var = vect_get_new_ssa_name (utype, vect_scalar_var);
3026 183 : tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
3027 183 : gassign *new_stmt
3028 183 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3029 183 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3030 183 : mask_arg = var;
3031 183 : if (!useless_type_conversion_p (masktype, utype))
3032 : {
3033 6 : gcc_assert (TYPE_PRECISION (utype)
3034 : <= TYPE_PRECISION (masktype));
3035 6 : var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3036 6 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3037 6 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3038 6 : mask_arg = var;
3039 : }
3040 183 : src_op = build_zero_cst (srctype);
3041 183 : mask_op = mask_arg;
3042 : }
3043 : else
3044 : {
3045 : src_op = mask;
3046 : mask_op = mask;
3047 : }
3048 : }
3049 : else
3050 : {
3051 154 : src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
3052 154 : mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
3053 : }
3054 :
3055 337 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3056 337 : gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
3057 : mask_op, scale);
3058 :
3059 337 : if (!useless_type_conversion_p (vectype, rettype))
3060 : {
3061 52 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
3062 : TYPE_VECTOR_SUBPARTS (rettype)));
3063 52 : op = vect_get_new_ssa_name (rettype, vect_simple_var);
3064 52 : gimple_call_set_lhs (new_stmt, op);
3065 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3066 52 : op = build1 (VIEW_CONVERT_EXPR, vectype, op);
3067 52 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
3068 : }
3069 :
3070 337 : return new_stmt;
3071 : }
3072 :
3073 : /* Build a scatter store call while vectorizing STMT_INFO. Insert new
3074 : instructions before GSI. GS_INFO describes the scatter store operation.
3075 : PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
3076 : vectorized data to store.
3077 : If the store is conditional, MASK is the vectorized condition, otherwise
3078 : MASK is null. */
3079 :
3080 : static gimple *
3081 161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
3082 : slp_tree slp_node,
3083 : gimple_stmt_iterator *gsi,
3084 : tree decl,
3085 : tree ptr, tree offset, tree oprnd, tree mask)
3086 : {
3087 161 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
3088 161 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
3089 161 : /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
3090 161 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3091 161 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3092 161 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3093 161 : tree scaletype = TREE_VALUE (arglist);
3094 161 : gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
3095 : && TREE_CODE (rettype) == VOID_TYPE);
3096 :
3097 161 : tree mask_arg = NULL_TREE;
3098 161 : if (mask)
3099 : {
3100 110 : mask_arg = mask;
3101 110 : tree optype = TREE_TYPE (mask_arg);
3102 110 : tree utype;
3103 110 : if (TYPE_MODE (masktype) == TYPE_MODE (optype))
3104 : utype = masktype;
3105 : else
3106 8 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3107 110 : tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
3108 110 : mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
3109 110 : gassign *new_stmt
3110 110 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3111 110 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3112 110 : mask_arg = var;
3113 110 : if (!useless_type_conversion_p (masktype, utype))
3114 : {
3115 8 : gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
3116 8 : tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3117 8 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3118 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3119 8 : mask_arg = var;
3120 : }
3121 : }
3122 : else
3123 : {
3124 51 : mask_arg = build_int_cst (masktype, -1);
3125 51 : mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
3126 : }
3127 :
3128 161 : tree src = oprnd;
3129 161 : if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
3130 : {
3131 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
3132 : TYPE_VECTOR_SUBPARTS (srctype)));
3133 0 : tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
3134 0 : src = build1 (VIEW_CONVERT_EXPR, srctype, src);
3135 0 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
3136 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3137 0 : src = var;
3138 : }
3139 :
3140 161 : tree op = offset;
3141 161 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3142 : {
3143 16 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3144 : TYPE_VECTOR_SUBPARTS (idxtype)));
3145 16 : tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3146 16 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3147 16 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3148 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3149 16 : op = var;
3150 : }
3151 :
3152 161 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3153 161 : gcall *new_stmt
3154 161 : = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
3155 161 : return new_stmt;
3156 : }
3157 :
3158 : /* Prepare the base and offset in GS_INFO for vectorization.
3159 : Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
3160 : to the vectorized offset argument for the first copy of STMT_INFO.
3161 : STMT_INFO is the statement described by GS_INFO and LOOP is the
3162 : containing loop. */
3163 :
3164 : static void
3165 1252 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
3166 : tree *dataref_ptr, vec<tree> *vec_offset)
3167 : {
3168 1252 : gimple_seq stmts = NULL;
3169 1252 : *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
3170 : &stmts, true, NULL_TREE);
3171 1252 : if (stmts != NULL)
3172 : {
3173 1023 : basic_block new_bb;
3174 1023 : edge pe = loop_preheader_edge (loop);
3175 1023 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3176 1023 : gcc_assert (!new_bb);
3177 : }
3178 1252 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
3179 1252 : }
3180 :
3181 : /* Prepare to implement a grouped or strided load or store using
3182 : the gather load or scatter store operation described by GS_INFO.
3183 : STMT_INFO is the load or store statement.
3184 :
3185 : Set *DR_STEP to the amount that should be added to pointer base address
3186 : to get to the next iteration's base address.
3187 : Set *DR_BUMP to the amount that should be added to the base
3188 : address after each copy of the vectorized statement in a grouped read.
3189 : Set *VEC_OFFSET to an invariant offset vector in which element I has the
3190 : value I * DR_STEP / SCALE. */
3191 :
3192 : static void
3193 0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
3194 : tree vectype, tree offset_vectype,
3195 : loop_vec_info loop_vinfo,
3196 : gimple_stmt_iterator *gsi,
3197 : tree *dr_step, tree *dr_bump,
3198 : tree *vec_offset)
3199 : {
3200 0 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
3201 :
3202 0 : tree dr_step_temp
3203 0 : = size_binop (MULT_EXPR,
3204 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3205 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
3206 0 : *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
3207 0 : ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
3208 0 : : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
3209 : true, GSI_SAME_STMT);
3210 0 : tree bump = size_binop (MULT_EXPR,
3211 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3212 : size_int (TYPE_VECTOR_SUBPARTS (vectype)));
3213 0 : *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
3214 :
3215 0 : internal_fn ifn
3216 0 : = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
3217 0 : if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
3218 : {
3219 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
3220 : unshare_expr (DR_STEP (dr)));
3221 0 : return;
3222 : }
3223 :
3224 : /* The offset given in GS_INFO can have pointer type, so use the element
3225 : type of the vector instead. */
3226 0 : tree offset_type = TREE_TYPE (offset_vectype);
3227 :
3228 : /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3229 0 : tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
3230 : ssize_int (SLP_TREE_GS_SCALE (node)));
3231 0 : step = fold_convert (offset_type, step);
3232 :
3233 : /* Create {0, X, X*2, X*3, ...}. */
3234 0 : tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
3235 : build_zero_cst (offset_type), step);
3236 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
3237 : }
3238 :
3239 : /* Return the amount that should be added to a vector pointer, represented by
3240 : DR_INFO, to increment to the next vectorized iteration. */
3241 :
3242 : static tree
3243 698568 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
3244 : vect_memory_access_type memory_access_type)
3245 : {
3246 698568 : if (memory_access_type == VMAT_INVARIANT)
3247 0 : return size_zero_node;
3248 :
3249 698568 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3250 :
3251 : /* For BB SLP there is no next iteration. */
3252 698568 : if (!loop_vinfo)
3253 559420 : return build_zero_cst (sizetype);
3254 :
3255 139148 : tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
3256 :
3257 : /* gather/scatter never reach here. */
3258 139148 : gcc_assert (!mat_gather_scatter_p (memory_access_type));
3259 :
3260 139148 : tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
3261 :
3262 139148 : return fold_build2 (MULT_EXPR, sizetype, iv_increment,
3263 : fold_convert (sizetype, step));
3264 : }
3265 :
3266 : /* Return the amount that should be added to a vector pointer to move
3267 : to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3268 : being vectorized and MEMORY_ACCESS_TYPE describes the type of
3269 : vectorization. */
3270 :
3271 : static tree
3272 699080 : vect_get_data_ptr_bump (vec_info *vinfo,
3273 : dr_vec_info *dr_info, tree aggr_type,
3274 : vect_memory_access_type memory_access_type)
3275 : {
3276 699080 : if (memory_access_type == VMAT_INVARIANT)
3277 0 : return size_zero_node;
3278 :
3279 699080 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3280 : /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
3281 : should never be needed. */
3282 139660 : if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
3283 : return NULL_TREE;
3284 :
3285 699080 : tree iv_step = TYPE_SIZE_UNIT (aggr_type);
3286 699080 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
3287 699080 : if (tree_int_cst_sgn (step) == -1)
3288 2899 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
3289 : return iv_step;
3290 : }
3291 :
3292 : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3293 :
3294 : static bool
3295 156 : vectorizable_bswap (vec_info *vinfo,
3296 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3297 : slp_tree slp_node,
3298 : slp_tree *slp_op,
3299 : tree vectype_in, stmt_vector_for_cost *cost_vec)
3300 : {
3301 156 : tree vectype = SLP_TREE_VECTYPE (slp_node);
3302 156 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3303 :
3304 156 : if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
3305 : {
3306 0 : if (dump_enabled_p ())
3307 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3308 : "mismatched vector sizes %T and %T\n",
3309 : vectype_in, vectype);
3310 : return false;
3311 : }
3312 :
3313 156 : tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
3314 156 : if (! char_vectype)
3315 : return false;
3316 :
3317 156 : poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
3318 156 : unsigned word_bytes;
3319 156 : if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
3320 : return false;
3321 :
3322 : /* The encoding uses one stepped pattern for each byte in the word. */
3323 156 : vec_perm_builder elts (num_bytes, word_bytes, 3);
3324 780 : for (unsigned i = 0; i < 3; ++i)
3325 2772 : for (unsigned j = 0; j < word_bytes; ++j)
3326 2304 : elts.quick_push ((i + 1) * word_bytes - j - 1);
3327 :
3328 156 : vec_perm_indices indices (elts, 1, num_bytes);
3329 156 : machine_mode vmode = TYPE_MODE (char_vectype);
3330 156 : if (!can_vec_perm_const_p (vmode, vmode, indices))
3331 : return false;
3332 :
3333 61 : if (cost_vec)
3334 : {
3335 45 : if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
3336 : {
3337 0 : if (dump_enabled_p ())
3338 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3339 : "incompatible vector types for invariants\n");
3340 : return false;
3341 : }
3342 :
3343 45 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3344 45 : DUMP_VECT_SCOPE ("vectorizable_bswap");
3345 45 : record_stmt_cost (cost_vec,
3346 : 1, vector_stmt, slp_node, 0, vect_prologue);
3347 45 : record_stmt_cost (cost_vec,
3348 45 : vect_get_num_copies (vinfo, slp_node),
3349 : vec_perm, slp_node, 0, vect_body);
3350 45 : return true;
3351 : }
3352 :
3353 16 : tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
3354 :
3355 : /* Transform. */
3356 16 : vec<tree> vec_oprnds = vNULL;
3357 16 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
3358 : /* Arguments are ready. create the new vector stmt. */
3359 16 : unsigned i;
3360 16 : tree vop;
3361 48 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
3362 : {
3363 16 : gimple *new_stmt;
3364 16 : tree tem = make_ssa_name (char_vectype);
3365 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3366 : char_vectype, vop));
3367 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3368 16 : tree tem2 = make_ssa_name (char_vectype);
3369 16 : new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
3370 : tem, tem, bswap_vconst);
3371 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3372 16 : tem = make_ssa_name (vectype);
3373 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3374 : vectype, tem2));
3375 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3376 16 : slp_node->push_vec_def (new_stmt);
3377 : }
3378 :
3379 16 : vec_oprnds.release ();
3380 16 : return true;
3381 156 : }
3382 :
3383 : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3384 : integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3385 : in a single step. On success, store the binary pack code in
3386 : *CONVERT_CODE. */
3387 :
3388 : static bool
3389 197 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
3390 : code_helper *convert_code)
3391 : {
3392 394 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
3393 394 : || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
3394 : return false;
3395 :
3396 81 : code_helper code;
3397 81 : int multi_step_cvt = 0;
3398 81 : auto_vec <tree, 8> interm_types;
3399 121 : if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
3400 : &code, &multi_step_cvt, &interm_types)
3401 81 : || multi_step_cvt)
3402 : return false;
3403 :
3404 41 : *convert_code = code;
3405 41 : return true;
3406 81 : }
3407 :
3408 : /* Function vectorizable_call.
3409 :
3410 : Check if STMT_INFO performs a function call that can be vectorized.
3411 : If COST_VEC is passed, calculate costs but don't change anything,
3412 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
3413 : it, and insert it at GSI.
3414 : Return true if STMT_INFO is vectorizable in this way. */
3415 :
3416 : static bool
3417 2678571 : vectorizable_call (vec_info *vinfo,
3418 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3419 : slp_tree slp_node,
3420 : stmt_vector_for_cost *cost_vec)
3421 : {
3422 2678571 : gcall *stmt;
3423 2678571 : tree vec_dest;
3424 2678571 : tree scalar_dest;
3425 2678571 : tree vec_oprnd0 = NULL_TREE;
3426 2678571 : tree vectype_out, vectype_in;
3427 2678571 : poly_uint64 nunits_in;
3428 2678571 : poly_uint64 nunits_out;
3429 2678571 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3430 2678571 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3431 2678571 : tree fndecl, new_temp, rhs_type;
3432 2678571 : enum vect_def_type dt[5]
3433 : = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
3434 : vect_unknown_def_type, vect_unknown_def_type };
3435 2678571 : tree vectypes[ARRAY_SIZE (dt)] = {};
3436 2678571 : slp_tree slp_op[ARRAY_SIZE (dt)] = {};
3437 2678571 : auto_vec<tree, 8> vargs;
3438 2678571 : enum { NARROW, NONE, WIDEN } modifier;
3439 2678571 : size_t i, nargs;
3440 2678571 : tree clz_ctz_arg1 = NULL_TREE;
3441 :
3442 2678571 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3443 : return false;
3444 :
3445 2678571 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3446 251571 : && cost_vec)
3447 : return false;
3448 :
3449 : /* Is STMT_INFO a vectorizable call? */
3450 2706651 : stmt = dyn_cast <gcall *> (stmt_info->stmt);
3451 28080 : if (!stmt)
3452 : return false;
3453 :
3454 28080 : if (gimple_call_internal_p (stmt)
3455 28080 : && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3456 17481 : || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3457 : /* Handled by vectorizable_load and vectorizable_store. */
3458 : return false;
3459 :
3460 24571 : if (gimple_call_lhs (stmt) == NULL_TREE
3461 24571 : || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3462 : return false;
3463 :
3464 24557 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
3465 :
3466 24557 : vectype_out = SLP_TREE_VECTYPE (slp_node);
3467 :
3468 : /* Process function arguments. */
3469 24557 : rhs_type = NULL_TREE;
3470 24557 : vectype_in = NULL_TREE;
3471 24557 : nargs = gimple_call_num_args (stmt);
3472 :
3473 : /* Bail out if the function has more than four arguments, we do not have
3474 : interesting builtin functions to vectorize with more than two arguments
3475 : except for fma (cond_fma has more). No arguments is also not good. */
3476 24557 : if (nargs == 0 || nargs > 5)
3477 : return false;
3478 :
3479 : /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3480 24477 : combined_fn cfn = gimple_call_combined_fn (stmt);
3481 24477 : if (cfn == CFN_GOMP_SIMD_LANE)
3482 : {
3483 3205 : nargs = 0;
3484 3205 : rhs_type = unsigned_type_node;
3485 : }
3486 : /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3487 : argument just says whether it is well-defined at zero or not and what
3488 : value should be returned for it. */
3489 24477 : if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
3490 : {
3491 194 : nargs = 1;
3492 194 : clz_ctz_arg1 = gimple_call_arg (stmt, 1);
3493 : }
3494 :
3495 24477 : int mask_opno = -1;
3496 24477 : if (internal_fn_p (cfn))
3497 : {
3498 : /* We can only handle direct internal masked calls here,
3499 : vectorizable_simd_clone_call is for the rest. */
3500 19744 : if (cfn == CFN_MASK_CALL)
3501 : return false;
3502 19573 : mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
3503 : }
3504 :
3505 75052 : for (i = 0; i < nargs; i++)
3506 : {
3507 52859 : if ((int) i == mask_opno)
3508 : {
3509 7926 : if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
3510 : &slp_op[i], &dt[i], &vectypes[i]))
3511 : return false;
3512 7926 : continue;
3513 : }
3514 :
3515 44933 : if (!vect_is_simple_use (vinfo, slp_node,
3516 : i, &slp_op[i], &dt[i], &vectypes[i]))
3517 : {
3518 0 : if (dump_enabled_p ())
3519 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3520 : "use not simple.\n");
3521 : return false;
3522 : }
3523 :
3524 : /* We can only handle calls with arguments of the same type. */
3525 44933 : tree op = gimple_call_arg (stmt, i);
3526 44933 : if (rhs_type
3527 44933 : && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3528 : {
3529 2113 : if (dump_enabled_p ())
3530 226 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3531 : "argument types differ.\n");
3532 : return false;
3533 : }
3534 42820 : if (!rhs_type)
3535 21101 : rhs_type = TREE_TYPE (op);
3536 :
3537 42820 : if (!vectype_in)
3538 22402 : vectype_in = vectypes[i];
3539 20418 : else if (vectypes[i]
3540 20418 : && !types_compatible_p (vectypes[i], vectype_in))
3541 : {
3542 0 : if (dump_enabled_p ())
3543 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3544 : "argument vector types differ.\n");
3545 : return false;
3546 : }
3547 : }
3548 : /* If all arguments are external or constant defs, infer the vector type
3549 : from the scalar type. */
3550 22193 : if (!vectype_in)
3551 5842 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
3552 22193 : if (!cost_vec)
3553 4611 : gcc_assert (vectype_in);
3554 17582 : if (!vectype_in)
3555 : {
3556 1102 : if (dump_enabled_p ())
3557 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3558 : "no vectype for scalar type %T\n", rhs_type);
3559 :
3560 : return false;
3561 : }
3562 :
3563 : /* During vect-patterns we remembered the defined-at-zero value in the
3564 : optional second argument for the natural vector mode. Now, once we have
3565 : the final vector mode, verify that they coincide. If not, bail out. */
3566 21091 : if (clz_ctz_arg1)
3567 : {
3568 194 : int val;
3569 194 : int d = (cfn == CFN_CLZ
3570 194 : ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val)
3571 199 : : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val));
3572 194 : if (d != 2 || val != tree_to_shwi (clz_ctz_arg1))
3573 : return false;
3574 : }
3575 :
3576 42130 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3577 21065 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
3578 : {
3579 12 : if (dump_enabled_p ())
3580 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3581 : "mixed mask and nonmask vector types\n");
3582 : return false;
3583 : }
3584 :
3585 21053 : if (vect_emulated_vector_p (vectype_in)
3586 21053 : || vect_emulated_vector_p (vectype_out))
3587 : {
3588 0 : if (dump_enabled_p ())
3589 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3590 : "use emulated vector type for call\n");
3591 : return false;
3592 : }
3593 :
3594 : /* FORNOW */
3595 21053 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3596 21053 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3597 21053 : if (known_eq (nunits_in * 2, nunits_out))
3598 : modifier = NARROW;
3599 20457 : else if (known_eq (nunits_out, nunits_in))
3600 : modifier = NONE;
3601 50 : else if (known_eq (nunits_out * 2, nunits_in))
3602 : modifier = WIDEN;
3603 : else
3604 : return false;
3605 :
3606 : /* We only handle functions that do not read or clobber memory. */
3607 42106 : if (gimple_vuse (stmt))
3608 : {
3609 1427 : if (dump_enabled_p ())
3610 18 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3611 : "function reads from or writes to memory.\n");
3612 : return false;
3613 : }
3614 :
3615 : /* For now, we only vectorize functions if a target specific builtin
3616 : is available. TODO -- in some cases, it might be profitable to
3617 : insert the calls for pieces of the vector, in order to be able
3618 : to vectorize other operations in the loop. */
3619 19626 : fndecl = NULL_TREE;
3620 19626 : internal_fn ifn = IFN_LAST;
3621 19626 : tree callee = gimple_call_fndecl (stmt);
3622 :
3623 : /* First try using an internal function. */
3624 19626 : code_helper convert_code = MAX_TREE_CODES;
3625 19626 : if (cfn != CFN_LAST
3626 19626 : && (modifier == NONE
3627 209 : || (modifier == NARROW
3628 197 : && simple_integer_narrowing (vectype_out, vectype_in,
3629 : &convert_code))))
3630 18575 : ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3631 : vectype_in);
3632 :
3633 : /* Check if the operation traps. */
3634 19626 : bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
3635 19626 : if (could_trap && cost_vec && loop_vinfo)
3636 : {
3637 : /* If the operation can trap it must be conditional, otherwise fail. */
3638 577 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3639 577 : ? ifn : get_conditional_internal_fn (ifn));
3640 577 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3641 577 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
3642 : {
3643 : /* We assume that BB SLP fills all lanes, so no inactive lanes can
3644 : cause issues. */
3645 136 : if ((cond_fn == IFN_LAST
3646 76 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3647 : OPTIMIZE_FOR_SPEED))
3648 212 : && (cond_len_fn == IFN_LAST
3649 76 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3650 : OPTIMIZE_FOR_SPEED)))
3651 : {
3652 136 : if (dump_enabled_p ())
3653 10 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3654 : "can't use a fully-masked loop because no"
3655 : " conditional operation is available.\n");
3656 136 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3657 : }
3658 : }
3659 : }
3660 :
3661 : /* If that fails, try asking for a target-specific built-in function. */
3662 19626 : if (ifn == IFN_LAST)
3663 : {
3664 11322 : if (cfn != CFN_LAST)
3665 10439 : fndecl = targetm.vectorize.builtin_vectorized_function
3666 10439 : (cfn, vectype_out, vectype_in);
3667 883 : else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
3668 24 : fndecl = targetm.vectorize.builtin_md_vectorized_function
3669 24 : (callee, vectype_out, vectype_in);
3670 : }
3671 :
3672 19626 : if (ifn == IFN_LAST && !fndecl)
3673 : {
3674 10179 : if (cfn == CFN_GOMP_SIMD_LANE
3675 3205 : && SLP_TREE_LANES (slp_node) == 1
3676 3205 : && loop_vinfo
3677 3205 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3678 3205 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3679 16589 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3680 3205 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3681 : {
3682 : /* We can handle IFN_GOMP_SIMD_LANE by returning a
3683 : { 0, 1, 2, ... vf - 1 } vector. */
3684 3205 : gcc_assert (nargs == 0);
3685 : }
3686 6974 : else if (modifier == NONE
3687 6974 : && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3688 6607 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3689 6538 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
3690 6485 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
3691 156 : return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
3692 156 : slp_op, vectype_in, cost_vec);
3693 : else
3694 : {
3695 6818 : if (dump_enabled_p ())
3696 301 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3697 : "function is not vectorizable.\n");
3698 : return false;
3699 : }
3700 : }
3701 :
3702 12652 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
3703 12652 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3704 12652 : ? ifn : get_conditional_internal_fn (ifn));
3705 12652 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3706 12652 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3707 10437 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
3708 12652 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
3709 12652 : if (cost_vec) /* transformation not required. */
3710 : {
3711 24954 : for (i = 0; i < nargs; ++i)
3712 16897 : if (!vect_maybe_update_slp_op_vectype (slp_op[i],
3713 16897 : vectypes[i]
3714 : ? vectypes[i] : vectype_in))
3715 : {
3716 0 : if (dump_enabled_p ())
3717 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3718 : "incompatible vector types for invariants\n");
3719 : return false;
3720 : }
3721 8057 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3722 8057 : DUMP_VECT_SCOPE ("vectorizable_call");
3723 8057 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
3724 :
3725 8057 : if (loop_vinfo
3726 6828 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
3727 4668 : && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3728 : {
3729 2618 : if (reduc_idx >= 0
3730 1631 : && (cond_fn == IFN_LAST
3731 1631 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3732 : OPTIMIZE_FOR_SPEED))
3733 2630 : && (cond_len_fn == IFN_LAST
3734 12 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3735 : OPTIMIZE_FOR_SPEED)))
3736 : {
3737 12 : if (dump_enabled_p ())
3738 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3739 : "can't use a fully-masked loop because no"
3740 : " conditional operation is available.\n");
3741 12 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3742 : }
3743 : else
3744 : {
3745 2606 : tree scalar_mask = NULL_TREE;
3746 2606 : if (mask_opno >= 0)
3747 2606 : scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3748 2606 : if (cond_len_fn != IFN_LAST
3749 2606 : && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3750 : OPTIMIZE_FOR_SPEED))
3751 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3752 : 1);
3753 : else
3754 2606 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3755 : scalar_mask);
3756 : }
3757 : }
3758 8057 : return true;
3759 : }
3760 :
3761 : /* Transform. */
3762 :
3763 4595 : if (dump_enabled_p ())
3764 428 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3765 :
3766 : /* Handle def. */
3767 4595 : scalar_dest = gimple_call_lhs (stmt);
3768 4595 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3769 :
3770 4595 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3771 3609 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
3772 4595 : unsigned int vect_nargs = nargs;
3773 4595 : if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3774 : {
3775 0 : ifn = cond_len_fn;
3776 : /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3777 0 : vect_nargs += 2;
3778 : /* But unless there's a mask argument already we need that
3779 : as well, and an else value. */
3780 0 : if (mask_opno == -1)
3781 0 : vect_nargs += 2;
3782 : }
3783 4595 : else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
3784 : {
3785 0 : ifn = cond_fn;
3786 0 : vect_nargs += 2;
3787 : }
3788 4595 : int len_opno = internal_fn_len_index (ifn);
3789 4595 : if (clz_ctz_arg1)
3790 59 : ++vect_nargs;
3791 :
3792 4595 : if (modifier == NONE || ifn != IFN_LAST)
3793 : {
3794 4563 : tree prev_res = NULL_TREE;
3795 4563 : vargs.safe_grow (vect_nargs, true);
3796 4563 : auto_vec<vec<tree> > vec_defs (nargs);
3797 :
3798 : /* Build argument list for the vectorized call. */
3799 4563 : if (cfn == CFN_GOMP_SIMD_LANE)
3800 : {
3801 3306 : for (i = 0; i < nvectors; ++i)
3802 : {
3803 : /* ??? For multi-lane SLP we'd need to build
3804 : { 0, 0, .., 1, 1, ... }. */
3805 1707 : tree cst = build_index_vector (vectype_out,
3806 : i * nunits_out, 1);
3807 1707 : tree new_var
3808 1707 : = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3809 1707 : gimple *init_stmt = gimple_build_assign (new_var, cst);
3810 1707 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
3811 1707 : new_temp = make_ssa_name (vec_dest);
3812 1707 : gimple *new_stmt = gimple_build_assign (new_temp, new_var);
3813 1707 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3814 1707 : slp_node->push_vec_def (new_stmt);
3815 : }
3816 : }
3817 : else
3818 : {
3819 2964 : vec<tree> vec_oprnds0;
3820 2964 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3821 2964 : vec_oprnds0 = vec_defs[0];
3822 :
3823 : /* Arguments are ready. Create the new vector stmt. */
3824 6134 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3825 : {
3826 3170 : int varg = 0;
3827 : /* Add the mask if necessary. */
3828 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3829 3172 : && internal_fn_mask_index (ifn) != -1)
3830 : {
3831 0 : gcc_assert (internal_fn_mask_index (ifn) == varg);
3832 0 : if (masked_loop_p)
3833 : {
3834 0 : unsigned int vec_num = vec_oprnds0.length ();
3835 0 : vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
3836 : masks, vec_num,
3837 : vectype_out, i);
3838 : }
3839 : else
3840 : {
3841 0 : tree mask_vectype = truth_type_for (vectype_out);
3842 0 : vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
3843 : stmt_info,
3844 : mask_vectype);
3845 : }
3846 : }
3847 3170 : size_t k;
3848 11307 : for (k = 0; k < nargs; k++)
3849 : {
3850 8137 : vec<tree> vec_oprndsk = vec_defs[k];
3851 8137 : vargs[varg++] = vec_oprndsk[i];
3852 : }
3853 : /* Add the else value if necessary. */
3854 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3855 3172 : && internal_fn_else_index (ifn) != -1)
3856 : {
3857 0 : gcc_assert (internal_fn_else_index (ifn) == varg);
3858 0 : if (reduc_idx >= 0)
3859 0 : vargs[varg++] = vargs[reduc_idx + 1];
3860 : else
3861 : {
3862 0 : auto else_value = targetm.preferred_else_value
3863 0 : (ifn, vectype_out, varg - 1, &vargs[1]);
3864 0 : vargs[varg++] = else_value;
3865 : }
3866 : }
3867 3170 : if (clz_ctz_arg1)
3868 59 : vargs[varg++] = clz_ctz_arg1;
3869 :
3870 3170 : gimple *new_stmt;
3871 3170 : if (modifier == NARROW)
3872 : {
3873 : /* We don't define any narrowing conditional functions
3874 : at present. */
3875 0 : gcc_assert (mask_opno < 0);
3876 0 : tree half_res = make_ssa_name (vectype_in);
3877 0 : gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3878 0 : gimple_call_set_lhs (call, half_res);
3879 0 : gimple_call_set_nothrow (call, true);
3880 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3881 0 : if ((i & 1) == 0)
3882 : {
3883 0 : prev_res = half_res;
3884 0 : continue;
3885 : }
3886 0 : new_temp = make_ssa_name (vec_dest);
3887 0 : new_stmt = vect_gimple_build (new_temp, convert_code,
3888 : prev_res, half_res);
3889 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3890 : }
3891 : else
3892 : {
3893 3170 : if (len_opno >= 0 && len_loop_p)
3894 : {
3895 0 : unsigned int vec_num = vec_oprnds0.length ();
3896 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
3897 : vec_num, vectype_out, i, 1, true);
3898 0 : signed char biasval
3899 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3900 0 : tree bias = build_int_cst (intQI_type_node, biasval);
3901 0 : vargs[len_opno] = len;
3902 0 : vargs[len_opno + 1] = bias;
3903 : }
3904 3170 : else if (mask_opno >= 0 && masked_loop_p)
3905 : {
3906 36 : unsigned int vec_num = vec_oprnds0.length ();
3907 36 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
3908 : vec_num, vectype_out, i);
3909 36 : vargs[mask_opno]
3910 72 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
3911 36 : vargs[mask_opno], gsi);
3912 : }
3913 :
3914 3170 : gcall *call;
3915 3170 : if (ifn != IFN_LAST)
3916 2861 : call = gimple_build_call_internal_vec (ifn, vargs);
3917 : else
3918 309 : call = gimple_build_call_vec (fndecl, vargs);
3919 3170 : new_temp = make_ssa_name (vec_dest, call);
3920 3170 : gimple_call_set_lhs (call, new_temp);
3921 3170 : gimple_call_set_nothrow (call, true);
3922 3170 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3923 3170 : new_stmt = call;
3924 : }
3925 3170 : slp_node->push_vec_def (new_stmt);
3926 : }
3927 : }
3928 :
3929 12123 : for (i = 0; i < nargs; i++)
3930 : {
3931 7560 : vec<tree> vec_oprndsi = vec_defs[i];
3932 7560 : vec_oprndsi.release ();
3933 : }
3934 4563 : }
3935 32 : else if (modifier == NARROW)
3936 : {
3937 32 : auto_vec<vec<tree> > vec_defs (nargs);
3938 : /* We don't define any narrowing conditional functions at present. */
3939 32 : gcc_assert (mask_opno < 0);
3940 :
3941 : /* Build argument list for the vectorized call. */
3942 32 : vargs.create (nargs * 2);
3943 :
3944 32 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3945 32 : vec<tree> vec_oprnds0 = vec_defs[0];
3946 :
3947 : /* Arguments are ready. Create the new vector stmt. */
3948 64 : for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3949 : {
3950 32 : size_t k;
3951 32 : vargs.truncate (0);
3952 96 : for (k = 0; k < nargs; k++)
3953 : {
3954 32 : vec<tree> vec_oprndsk = vec_defs[k];
3955 32 : vargs.quick_push (vec_oprndsk[i]);
3956 32 : vargs.quick_push (vec_oprndsk[i + 1]);
3957 : }
3958 32 : gcall *call;
3959 32 : if (ifn != IFN_LAST)
3960 : call = gimple_build_call_internal_vec (ifn, vargs);
3961 : else
3962 32 : call = gimple_build_call_vec (fndecl, vargs);
3963 32 : new_temp = make_ssa_name (vec_dest, call);
3964 32 : gimple_call_set_lhs (call, new_temp);
3965 32 : gimple_call_set_nothrow (call, true);
3966 32 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3967 32 : slp_node->push_vec_def (call);
3968 : }
3969 :
3970 64 : for (i = 0; i < nargs; i++)
3971 : {
3972 32 : vec<tree> vec_oprndsi = vec_defs[i];
3973 32 : vec_oprndsi.release ();
3974 : }
3975 32 : }
3976 : else
3977 : /* No current target implements this case. */
3978 : return false;
3979 :
3980 4595 : vargs.release ();
3981 :
3982 4595 : return true;
3983 2678571 : }
3984 :
3985 :
3986 : struct simd_call_arg_info
3987 : {
3988 : tree vectype;
3989 : tree op;
3990 : HOST_WIDE_INT linear_step;
3991 : enum vect_def_type dt;
3992 : unsigned int align;
3993 : bool simd_lane_linear;
3994 : };
3995 :
3996 : /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3997 : is linear within simd lane (but not within whole loop), note it in
3998 : *ARGINFO. */
3999 :
4000 : static void
4001 19 : vect_simd_lane_linear (tree op, class loop *loop,
4002 : struct simd_call_arg_info *arginfo)
4003 : {
4004 19 : gimple *def_stmt = SSA_NAME_DEF_STMT (op);
4005 :
4006 19 : if (!is_gimple_assign (def_stmt)
4007 19 : || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
4008 35 : || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
4009 : return;
4010 :
4011 16 : tree base = gimple_assign_rhs1 (def_stmt);
4012 16 : HOST_WIDE_INT linear_step = 0;
4013 16 : tree v = gimple_assign_rhs2 (def_stmt);
4014 64 : while (TREE_CODE (v) == SSA_NAME)
4015 : {
4016 48 : tree t;
4017 48 : def_stmt = SSA_NAME_DEF_STMT (v);
4018 48 : if (is_gimple_assign (def_stmt))
4019 32 : switch (gimple_assign_rhs_code (def_stmt))
4020 : {
4021 0 : case PLUS_EXPR:
4022 0 : t = gimple_assign_rhs2 (def_stmt);
4023 0 : if (linear_step || TREE_CODE (t) != INTEGER_CST)
4024 : return;
4025 0 : base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
4026 0 : v = gimple_assign_rhs1 (def_stmt);
4027 0 : continue;
4028 16 : case MULT_EXPR:
4029 16 : t = gimple_assign_rhs2 (def_stmt);
4030 16 : if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
4031 : return;
4032 16 : linear_step = tree_to_shwi (t);
4033 16 : v = gimple_assign_rhs1 (def_stmt);
4034 16 : continue;
4035 16 : CASE_CONVERT:
4036 16 : t = gimple_assign_rhs1 (def_stmt);
4037 16 : if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
4038 16 : || (TYPE_PRECISION (TREE_TYPE (v))
4039 16 : < TYPE_PRECISION (TREE_TYPE (t))))
4040 : return;
4041 16 : if (!linear_step)
4042 0 : linear_step = 1;
4043 16 : v = t;
4044 16 : continue;
4045 : default:
4046 : return;
4047 : }
4048 16 : else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
4049 16 : && loop->simduid
4050 12 : && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
4051 28 : && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
4052 : == loop->simduid))
4053 : {
4054 12 : if (!linear_step)
4055 0 : linear_step = 1;
4056 12 : arginfo->linear_step = linear_step;
4057 12 : arginfo->op = base;
4058 12 : arginfo->simd_lane_linear = true;
4059 : }
4060 : return;
4061 : }
4062 : }
4063 :
4064 : /* Function vectorizable_simd_clone_call.
4065 :
4066 : Check if STMT_INFO performs a function call that can be vectorized
4067 : by calling a simd clone of the function.
4068 : If COST_VEC is passed, calculate costs but don't change anything,
4069 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
4070 : it, and insert it at GSI.
4071 : Return true if STMT_INFO is vectorizable in this way. */
4072 :
4073 : static bool
4074 2666240 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
4075 : gimple_stmt_iterator *gsi,
4076 : slp_tree slp_node,
4077 : stmt_vector_for_cost *cost_vec)
4078 : {
4079 2666240 : tree vec_dest;
4080 2666240 : tree scalar_dest;
4081 2666240 : tree vec_oprnd0 = NULL_TREE;
4082 2666240 : tree vectype;
4083 2666240 : poly_uint64 nunits;
4084 2666240 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
4085 2666240 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
4086 2666240 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
4087 2666240 : tree fndecl, new_temp;
4088 2666240 : int j;
4089 2666240 : auto_vec<simd_call_arg_info> arginfo;
4090 2666240 : vec<tree> vargs = vNULL;
4091 2666240 : size_t i, nargs;
4092 2666240 : tree rtype, ratype;
4093 2666240 : vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
4094 2666240 : int masked_call_offset = 0;
4095 :
4096 : /* Is STMT a vectorizable call? */
4097 2666240 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
4098 16932 : if (!stmt)
4099 : return false;
4100 :
4101 16932 : fndecl = gimple_call_fndecl (stmt);
4102 16932 : if (fndecl == NULL_TREE
4103 16932 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
4104 : {
4105 240 : fndecl = gimple_call_arg (stmt, 0);
4106 240 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
4107 240 : fndecl = TREE_OPERAND (fndecl, 0);
4108 240 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
4109 : masked_call_offset = 1;
4110 : }
4111 16692 : if (fndecl == NULL_TREE)
4112 : return false;
4113 :
4114 6570 : struct cgraph_node *node = cgraph_node::get (fndecl);
4115 6570 : if (node == NULL || node->simd_clones == NULL)
4116 : return false;
4117 :
4118 1556 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4119 : return false;
4120 :
4121 1556 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4122 0 : && cost_vec)
4123 : return false;
4124 :
4125 1556 : if (gimple_call_lhs (stmt)
4126 1556 : && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
4127 : return false;
4128 :
4129 1556 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
4130 :
4131 1556 : vectype = SLP_TREE_VECTYPE (slp_node);
4132 :
4133 2667796 : if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
4134 : return false;
4135 :
4136 : /* Process function arguments. */
4137 1556 : nargs = gimple_call_num_args (stmt) - masked_call_offset;
4138 :
4139 : /* Bail out if the function has zero arguments. */
4140 1556 : if (nargs == 0)
4141 : return false;
4142 :
4143 1492 : vect_simd_clone_data _data;
4144 1492 : vect_simd_clone_data &data = slp_node->get_data (_data);
4145 1492 : vec<tree>& simd_clone_info = data.simd_clone_info;
4146 1492 : arginfo.reserve (nargs, true);
4147 1492 : auto_vec<slp_tree> slp_op;
4148 1492 : slp_op.safe_grow_cleared (nargs);
4149 :
4150 5791 : for (i = 0; i < nargs; i++)
4151 : {
4152 2815 : simd_call_arg_info thisarginfo;
4153 2815 : affine_iv iv;
4154 2815 : tree op;
4155 :
4156 2815 : thisarginfo.linear_step = 0;
4157 2815 : thisarginfo.align = 0;
4158 2815 : thisarginfo.op = NULL_TREE;
4159 2815 : thisarginfo.simd_lane_linear = false;
4160 :
4161 5630 : int op_no = vect_slp_child_index_for_operand (stmt_info,
4162 2815 : i + masked_call_offset);
4163 5630 : if (!vect_is_simple_use (vinfo, slp_node,
4164 2815 : op_no, &op, &slp_op[i],
4165 : &thisarginfo.dt, &thisarginfo.vectype)
4166 2815 : || thisarginfo.dt == vect_uninitialized_def)
4167 : {
4168 0 : if (dump_enabled_p ())
4169 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4170 : "use not simple.\n");
4171 8 : return false;
4172 : }
4173 :
4174 2815 : if (thisarginfo.dt == vect_constant_def
4175 2815 : || thisarginfo.dt == vect_external_def)
4176 : {
4177 : /* With SLP we determine the vector type of constants/externals
4178 : at analysis time, handling conflicts via
4179 : vect_maybe_update_slp_op_vectype. At transform time
4180 : we have a vector type recorded for SLP. */
4181 690 : gcc_assert (cost_vec
4182 : || thisarginfo.vectype != NULL_TREE);
4183 : if (cost_vec)
4184 : {
4185 558 : thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
4186 558 : TREE_TYPE (op),
4187 : slp_node);
4188 : /* ??? While for example uniform arguments might not need
4189 : a vector type we are not set up to handle this case
4190 : right now. */
4191 558 : if (!thisarginfo.vectype)
4192 : {
4193 8 : if (dump_enabled_p ())
4194 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4195 : "no vector type for argument.\n");
4196 : return false;
4197 : }
4198 : }
4199 : }
4200 : else
4201 2125 : gcc_assert (thisarginfo.vectype != NULL_TREE);
4202 :
4203 : /* For linear arguments, the analyze phase should have saved
4204 : the base and step. */
4205 2675 : if (!cost_vec
4206 1666 : && i * 3 + 4 <= simd_clone_info.length ()
4207 2885 : && simd_clone_info[i * 3 + 2])
4208 : {
4209 118 : thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
4210 118 : thisarginfo.op = simd_clone_info[i * 3 + 1];
4211 118 : thisarginfo.simd_lane_linear
4212 118 : = (simd_clone_info[i * 3 + 3] == boolean_true_node);
4213 : /* If loop has been peeled for alignment, we need to adjust it. */
4214 118 : tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
4215 118 : tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
4216 118 : if (n1 != n2 && !thisarginfo.simd_lane_linear)
4217 : {
4218 0 : tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
4219 0 : tree step = simd_clone_info[i * 3 + 2];
4220 0 : tree opt = TREE_TYPE (thisarginfo.op);
4221 0 : bias = fold_convert (TREE_TYPE (step), bias);
4222 0 : bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
4223 0 : thisarginfo.op
4224 0 : = fold_build2 (POINTER_TYPE_P (opt)
4225 : ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
4226 : thisarginfo.op, bias);
4227 : }
4228 : }
4229 2689 : else if (cost_vec
4230 1974 : && thisarginfo.dt != vect_constant_def
4231 1846 : && thisarginfo.dt != vect_external_def
4232 1424 : && loop_vinfo
4233 1407 : && SLP_TREE_LANES (slp_node) == 1
4234 1383 : && TREE_CODE (op) == SSA_NAME
4235 2766 : && simple_iv (loop, loop_containing_stmt (stmt), op,
4236 : &iv, false)
4237 2905 : && tree_fits_shwi_p (iv.step))
4238 : {
4239 216 : thisarginfo.linear_step = tree_to_shwi (iv.step);
4240 216 : thisarginfo.op = iv.base;
4241 : }
4242 2473 : else if ((thisarginfo.dt == vect_constant_def
4243 2473 : || thisarginfo.dt == vect_external_def)
4244 682 : && SLP_TREE_LANES (slp_node) == 1
4245 2781 : && POINTER_TYPE_P (TREE_TYPE (op)))
4246 86 : thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
4247 : /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4248 : linear too. */
4249 2807 : if (SLP_TREE_LANES (slp_node) == 1
4250 2363 : && POINTER_TYPE_P (TREE_TYPE (op))
4251 200 : && !thisarginfo.linear_step
4252 116 : && cost_vec
4253 62 : && thisarginfo.dt != vect_constant_def
4254 62 : && thisarginfo.dt != vect_external_def
4255 19 : && loop_vinfo
4256 2826 : && TREE_CODE (op) == SSA_NAME)
4257 19 : vect_simd_lane_linear (op, loop, &thisarginfo);
4258 :
4259 2807 : if (!vectype)
4260 12 : vectype = thisarginfo.vectype;
4261 2807 : arginfo.quick_push (thisarginfo);
4262 : }
4263 :
4264 1484 : poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
4265 1484 : unsigned group_size = SLP_TREE_LANES (slp_node);
4266 1484 : unsigned int badness = 0;
4267 1484 : unsigned int badness_inbranch = 0;
4268 1484 : struct cgraph_node *bestn = NULL;
4269 1484 : struct cgraph_node *bestn_inbranch = NULL;
4270 1484 : if (!cost_vec)
4271 382 : bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4272 382 : ? data.clone_inbranch : data.clone);
4273 : else
4274 6302 : for (struct cgraph_node *n = node->simd_clones; n != NULL;
4275 5200 : n = n->simdclone->next_clone)
4276 : {
4277 5200 : unsigned int this_badness = 0;
4278 5200 : unsigned int num_calls;
4279 : /* The number of arguments in the call and the number of parameters in
4280 : the simdclone should match. However, when the simdclone is
4281 : 'inbranch', it could have one more parameter than nargs when using
4282 : an inbranch simdclone to call a non-inbranch call, either in a
4283 : non-masked loop using a all true constant mask, or inside a masked
4284 : loop using it's mask. */
4285 5200 : size_t simd_nargs = n->simdclone->nargs;
4286 5200 : if (!masked_call_offset && n->simdclone->inbranch)
4287 2291 : simd_nargs--;
4288 5200 : if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
4289 : &num_calls)
4290 2071 : || (!n->simdclone->inbranch && (masked_call_offset > 0))
4291 1863 : || (nargs != simd_nargs))
4292 3337 : continue;
4293 1863 : if (num_calls != 1)
4294 1276 : this_badness += floor_log2 (num_calls) * 4096;
4295 1863 : if (n->simdclone->inbranch)
4296 801 : this_badness += 8192;
4297 :
4298 : /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
4299 : mode, which for targets that use it will determine what ISA we can
4300 : vectorize this code with. */
4301 1863 : machine_mode vector_mode = vinfo->vector_mode;
4302 1863 : if (vectype)
4303 1863 : vector_mode = TYPE_MODE (vectype);
4304 1863 : int target_badness = targetm.simd_clone.usable (n, vector_mode);
4305 1863 : if (target_badness < 0)
4306 390 : continue;
4307 1473 : this_badness += target_badness * 512;
4308 4376 : for (i = 0; i < nargs; i++)
4309 : {
4310 3178 : switch (n->simdclone->args[i].arg_type)
4311 : {
4312 2216 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4313 2216 : if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
4314 : /* Vector mask arguments are not supported. */
4315 : i = -1;
4316 2208 : else if (!useless_type_conversion_p
4317 2208 : (n->simdclone->args[i].orig_type,
4318 2208 : TREE_TYPE (gimple_call_arg (stmt,
4319 : i + masked_call_offset))))
4320 : i = -1;
4321 2208 : else if (arginfo[i].dt == vect_constant_def
4322 2100 : || arginfo[i].dt == vect_external_def
4323 4244 : || arginfo[i].linear_step)
4324 400 : this_badness += 64;
4325 : break;
4326 310 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4327 310 : if ((arginfo[i].dt != vect_constant_def
4328 145 : && arginfo[i].dt != vect_external_def)
4329 410 : || SLP_TREE_LANES (slp_node) != 1)
4330 : i = -1;
4331 : break;
4332 328 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4333 328 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4334 328 : if (arginfo[i].dt == vect_constant_def
4335 328 : || arginfo[i].dt == vect_external_def
4336 328 : || (arginfo[i].linear_step
4337 328 : != n->simdclone->args[i].linear_step))
4338 : i = -1;
4339 : break;
4340 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4341 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4342 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4343 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4344 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4345 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4346 : /* FORNOW */
4347 : i = -1;
4348 : break;
4349 324 : case SIMD_CLONE_ARG_TYPE_MASK:
4350 324 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4351 285 : && n->simdclone->mask_mode != VOIDmode)
4352 : i = -1;
4353 : /* While we can create a traditional data vector from
4354 : an incoming integer mode mask we have no good way to
4355 : force generate an integer mode mask from a traditional
4356 : boolean vector input. */
4357 324 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4358 324 : && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4359 : i = -1;
4360 318 : else if (n->simdclone->mask_mode == VOIDmode
4361 : /* FORNOW we only have partial support for vector-type
4362 : masks that can't hold all of simdlen. */
4363 603 : && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
4364 467 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
4365 : /* Verify we can compute the mask argument. */
4366 115 : || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
4367 115 : arginfo[i].vectype)))
4368 : i = -1;
4369 136 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4370 : /* FORNOW we only have partial support for
4371 : integer-type masks that represent the same number
4372 : of lanes as the vectorized mask inputs. */
4373 169 : && maybe_ne (exact_div (n->simdclone->simdlen,
4374 : n->simdclone->args[i].linear_step),
4375 33 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4376 : i = -1;
4377 112 : else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4378 112 : && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4379 12 : this_badness += 2048;
4380 : break;
4381 : }
4382 24 : if (i == (size_t) -1)
4383 : break;
4384 2903 : if (n->simdclone->args[i].alignment > arginfo[i].align)
4385 : {
4386 : i = -1;
4387 : break;
4388 : }
4389 2903 : if (arginfo[i].align)
4390 110 : this_badness += (exact_log2 (arginfo[i].align)
4391 160 : - exact_log2 (n->simdclone->args[i].alignment));
4392 : }
4393 1473 : if (i == (size_t) -1)
4394 275 : continue;
4395 1198 : if (masked_call_offset == 0
4396 1086 : && n->simdclone->inbranch
4397 351 : && n->simdclone->nargs > nargs)
4398 : {
4399 351 : gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
4400 : SIMD_CLONE_ARG_TYPE_MASK);
4401 351 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4402 335 : && n->simdclone->mask_mode != VOIDmode)
4403 0 : continue;
4404 : /* Penalize using a masked SIMD clone in a non-masked loop, that is
4405 : not in a branch, as we'd have to construct an all-true mask. */
4406 351 : this_badness += 64;
4407 : }
4408 1198 : if (bestn == NULL || this_badness < badness)
4409 : {
4410 861 : bestn = n;
4411 861 : badness = this_badness;
4412 : }
4413 1198 : if (n->simdclone->inbranch
4414 463 : && (bestn_inbranch == NULL || this_badness < badness_inbranch))
4415 : {
4416 5200 : bestn_inbranch = n;
4417 5200 : badness_inbranch = this_badness;
4418 : }
4419 : }
4420 :
4421 1484 : if (bestn == NULL)
4422 : return false;
4423 :
4424 883 : fndecl = bestn->decl;
4425 883 : nunits = bestn->simdclone->simdlen;
4426 883 : int ncopies = vector_unroll_factor (vf * group_size, nunits);
4427 :
4428 : /* If the function isn't const, only allow it in simd loops where user
4429 : has asserted that at least nunits consecutive iterations can be
4430 : performed using SIMD instructions. */
4431 866 : if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
4432 1072 : && gimple_vuse (stmt))
4433 : return false;
4434 :
4435 : /* ncopies is the number of SIMD clone calls we create, since simdlen
4436 : is not necessarily matching nunits of the vector types used, track
4437 : that in ncopies_in. */
4438 883 : int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
4439 :
4440 : /* Sanity check: make sure that at least one copy of the vectorized stmt
4441 : needs to be generated. */
4442 883 : gcc_assert (ncopies >= 1);
4443 :
4444 883 : if (cost_vec) /* transformation not required. */
4445 : {
4446 1642 : for (unsigned i = 0; i < nargs; ++i)
4447 1141 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
4448 : {
4449 0 : if (dump_enabled_p ())
4450 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4451 : "incompatible vector types for invariants\n");
4452 : return false;
4453 : }
4454 :
4455 501 : if (!bestn_inbranch && loop_vinfo)
4456 : {
4457 271 : if (dump_enabled_p ()
4458 271 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
4459 193 : dump_printf_loc (MSG_NOTE, vect_location,
4460 : "can't use a fully-masked loop because no"
4461 : " masked simd clone was available.\n");
4462 271 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4463 : }
4464 :
4465 : /* When the original call is pure or const but the SIMD ABI dictates
4466 : an aggregate return we will have to use a virtual definition and
4467 : in a loop eventually even need to add a virtual PHI. That's
4468 : not straight-forward so allow to fix this up via renaming. */
4469 501 : if (gimple_call_lhs (stmt)
4470 495 : && !gimple_vdef (stmt)
4471 888 : && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
4472 36 : vinfo->any_known_not_updated_vssa = true;
4473 : /* ??? For SLP code-gen we end up inserting after the last
4474 : vector argument def rather than at the original call position
4475 : so automagic virtual operand updating doesn't work. */
4476 1002 : if (gimple_vuse (stmt))
4477 151 : vinfo->any_known_not_updated_vssa = true;
4478 :
4479 501 : data.clone = bestn;
4480 501 : data.clone_inbranch = bestn_inbranch;
4481 :
4482 501 : simd_clone_info.safe_push (NULL_TREE);
4483 1799 : for (i = 0;
4484 2729 : i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
4485 : {
4486 1298 : if (loop_vinfo
4487 1274 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
4488 480 : && (bestn_inbranch->simdclone->args[i].arg_type
4489 : == SIMD_CLONE_ARG_TYPE_MASK))
4490 : {
4491 173 : if (masked_call_offset)
4492 : /* When there is an explicit mask we require the
4493 : number of elements to match up. */
4494 52 : vect_record_loop_mask (loop_vinfo,
4495 : &LOOP_VINFO_MASKS (loop_vinfo),
4496 : ncopies_in, vectype, NULL_TREE);
4497 : else
4498 : {
4499 : /* When there is no explicit mask on the call we have
4500 : more relaxed requirements. */
4501 121 : tree masktype;
4502 121 : poly_uint64 callee_nelements;
4503 121 : if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
4504 : {
4505 16 : callee_nelements
4506 16 : = exact_div (bestn_inbranch->simdclone->simdlen,
4507 : bestn_inbranch->simdclone->args[i].linear_step);
4508 16 : masktype = get_related_vectype_for_scalar_type
4509 16 : (vinfo->vector_mode, TREE_TYPE (vectype),
4510 : callee_nelements);
4511 : }
4512 : else
4513 : {
4514 105 : masktype = bestn_inbranch->simdclone->args[i].vector_type;
4515 105 : if (VECTOR_BOOLEAN_TYPE_P (masktype))
4516 : {
4517 0 : if (VECTOR_MODE_P (TYPE_MODE (masktype)))
4518 : /* The aarch64 port will add custom attributes to
4519 : types for SVE simdclones which make the types
4520 : different. We should use canonincal types for
4521 : masks within the vectorizer, hence we
4522 : re-construct the vectype from its mode. */
4523 0 : masktype = build_truth_vector_type_for_mode
4524 0 : (TYPE_VECTOR_SUBPARTS (masktype),
4525 0 : TYPE_MODE (masktype));
4526 : }
4527 : else
4528 105 : masktype = truth_type_for (masktype);
4529 105 : callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
4530 : }
4531 121 : if (!masktype)
4532 : {
4533 0 : if (dump_enabled_p ())
4534 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4535 : "can't use a fully-masked loop because"
4536 : " no suitable vector type was found for"
4537 : " the mask.\n");
4538 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4539 : }
4540 : else
4541 : {
4542 121 : auto o = vector_unroll_factor (nunits, callee_nelements);
4543 121 : vect_record_loop_mask (loop_vinfo,
4544 : &LOOP_VINFO_MASKS (loop_vinfo),
4545 : ncopies * o, masktype, NULL_TREE);
4546 : }
4547 : }
4548 : }
4549 1125 : else if ((bestn->simdclone->args[i].arg_type
4550 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4551 1018 : || (bestn->simdclone->args[i].arg_type
4552 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
4553 1007 : || (bestn_inbranch
4554 399 : && ((bestn_inbranch->simdclone->args[i].arg_type
4555 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4556 399 : || (bestn_inbranch->simdclone->args[i].arg_type
4557 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
4558 : {
4559 118 : simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
4560 118 : simd_clone_info.safe_push (arginfo[i].op);
4561 202 : tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
4562 202 : ? size_type_node : TREE_TYPE (arginfo[i].op));
4563 118 : tree ls = build_int_cst (lst, arginfo[i].linear_step);
4564 118 : simd_clone_info.safe_push (ls);
4565 118 : tree sll = (arginfo[i].simd_lane_linear
4566 118 : ? boolean_true_node : boolean_false_node);
4567 118 : simd_clone_info.safe_push (sll);
4568 : }
4569 : }
4570 :
4571 501 : SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
4572 501 : slp_node->data = new vect_simd_clone_data (std::move (_data));
4573 501 : DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4574 : /* ??? We're confused by calls w/o LHS. */
4575 501 : if (SLP_TREE_VECTYPE (slp_node))
4576 495 : vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
4577 501 : return true;
4578 : }
4579 :
4580 : /* Transform. */
4581 :
4582 382 : if (dump_enabled_p ())
4583 259 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
4584 :
4585 : /* Handle def. */
4586 382 : scalar_dest = gimple_call_lhs (stmt);
4587 382 : vec_dest = NULL_TREE;
4588 382 : rtype = NULL_TREE;
4589 382 : ratype = NULL_TREE;
4590 382 : if (scalar_dest)
4591 : {
4592 376 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
4593 376 : rtype = TREE_TYPE (TREE_TYPE (fndecl));
4594 376 : if (TREE_CODE (rtype) == ARRAY_TYPE)
4595 : {
4596 12 : ratype = rtype;
4597 12 : rtype = TREE_TYPE (ratype);
4598 : }
4599 : }
4600 :
4601 764 : auto_vec<vec<tree> > vec_oprnds;
4602 382 : auto_vec<unsigned> vec_oprnds_i;
4603 382 : vec_oprnds_i.safe_grow_cleared (nargs, true);
4604 382 : vec_oprnds.reserve_exact (nargs);
4605 382 : vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
4606 1284 : for (j = 0; j < ncopies; ++j)
4607 : {
4608 520 : poly_uint64 callee_nelements;
4609 520 : poly_uint64 caller_nelements;
4610 : /* Build argument list for the vectorized call. */
4611 520 : if (j == 0)
4612 382 : vargs.create (nargs);
4613 : else
4614 138 : vargs.truncate (0);
4615 :
4616 1697 : for (i = 0; i < nargs; i++)
4617 : {
4618 1177 : unsigned int k, l, m, o;
4619 1177 : tree atype;
4620 1177 : tree op = gimple_call_arg (stmt, i + masked_call_offset);
4621 1177 : switch (bestn->simdclone->args[i].arg_type)
4622 : {
4623 883 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4624 883 : atype = bestn->simdclone->args[i].vector_type;
4625 883 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4626 883 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4627 883 : o = vector_unroll_factor (nunits, callee_nelements);
4628 2001 : for (m = j * o; m < (j + 1) * o; m++)
4629 : {
4630 1118 : if (known_lt (callee_nelements, caller_nelements))
4631 : {
4632 596 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
4633 298 : if (!constant_multiple_p (caller_nelements,
4634 : callee_nelements, &k))
4635 0 : gcc_unreachable ();
4636 :
4637 298 : gcc_assert ((k & (k - 1)) == 0);
4638 298 : if (m == 0)
4639 : {
4640 65 : vec_oprnds_i[i] = 0;
4641 65 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4642 : }
4643 : else
4644 : {
4645 233 : vec_oprnd0 = arginfo[i].op;
4646 233 : if ((m & (k - 1)) == 0)
4647 78 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4648 : }
4649 298 : arginfo[i].op = vec_oprnd0;
4650 298 : vec_oprnd0
4651 298 : = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
4652 298 : bitsize_int (prec),
4653 298 : bitsize_int ((m & (k - 1)) * prec));
4654 298 : gassign *new_stmt
4655 298 : = gimple_build_assign (make_ssa_name (atype),
4656 : vec_oprnd0);
4657 298 : vect_finish_stmt_generation (vinfo, stmt_info,
4658 : new_stmt, gsi);
4659 298 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4660 : }
4661 : else
4662 : {
4663 820 : if (!constant_multiple_p (callee_nelements,
4664 : caller_nelements, &k))
4665 0 : gcc_unreachable ();
4666 820 : gcc_assert ((k & (k - 1)) == 0);
4667 820 : vec<constructor_elt, va_gc> *ctor_elts;
4668 820 : if (k != 1)
4669 12 : vec_alloc (ctor_elts, k);
4670 : else
4671 808 : ctor_elts = NULL;
4672 844 : for (l = 0; l < k; l++)
4673 : {
4674 832 : if (m == 0 && l == 0)
4675 : {
4676 479 : vec_oprnds_i[i] = 0;
4677 479 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4678 : }
4679 : else
4680 353 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4681 832 : arginfo[i].op = vec_oprnd0;
4682 832 : if (k == 1)
4683 : break;
4684 24 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
4685 : vec_oprnd0);
4686 : }
4687 820 : if (k == 1)
4688 808 : if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
4689 : atype))
4690 : {
4691 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
4692 : vec_oprnd0);
4693 0 : gassign *new_stmt
4694 0 : = gimple_build_assign (make_ssa_name (atype),
4695 : vec_oprnd0);
4696 0 : vect_finish_stmt_generation (vinfo, stmt_info,
4697 : new_stmt, gsi);
4698 0 : vargs.safe_push (gimple_get_lhs (new_stmt));
4699 : }
4700 : else
4701 808 : vargs.safe_push (vec_oprnd0);
4702 : else
4703 : {
4704 12 : vec_oprnd0 = build_constructor (atype, ctor_elts);
4705 12 : gassign *new_stmt
4706 12 : = gimple_build_assign (make_ssa_name (atype),
4707 : vec_oprnd0);
4708 12 : vect_finish_stmt_generation (vinfo, stmt_info,
4709 : new_stmt, gsi);
4710 12 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4711 : }
4712 : }
4713 : }
4714 : break;
4715 71 : case SIMD_CLONE_ARG_TYPE_MASK:
4716 71 : if (bestn->simdclone->mask_mode == VOIDmode)
4717 : {
4718 64 : atype = bestn->simdclone->args[i].vector_type;
4719 64 : tree elt_type = TREE_TYPE (atype);
4720 64 : tree one = fold_convert (elt_type, integer_one_node);
4721 64 : tree zero = fold_convert (elt_type, integer_zero_node);
4722 64 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4723 64 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4724 64 : o = vector_unroll_factor (nunits, callee_nelements);
4725 132 : for (m = j * o; m < (j + 1) * o; m++)
4726 : {
4727 68 : if (maybe_lt (callee_nelements, caller_nelements))
4728 : {
4729 : /* The mask type has fewer elements than simdlen. */
4730 :
4731 : /* FORNOW */
4732 0 : gcc_unreachable ();
4733 : }
4734 68 : else if (known_eq (callee_nelements, caller_nelements))
4735 : {
4736 : /* The SIMD clone function has the same number of
4737 : elements as the current function. */
4738 68 : if (m == 0)
4739 62 : vec_oprnds_i[i] = 0;
4740 68 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4741 68 : if (loop_vinfo
4742 68 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4743 : {
4744 0 : vec_loop_masks *loop_masks
4745 : = &LOOP_VINFO_MASKS (loop_vinfo);
4746 0 : tree loop_mask
4747 0 : = vect_get_loop_mask (loop_vinfo, gsi,
4748 : loop_masks, ncopies_in,
4749 0 : vectype, m);
4750 0 : vec_oprnd0
4751 0 : = prepare_vec_mask (loop_vinfo,
4752 0 : TREE_TYPE (loop_mask),
4753 : loop_mask, vec_oprnd0,
4754 : gsi);
4755 0 : loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
4756 : loop_mask });
4757 :
4758 : }
4759 68 : vec_oprnd0
4760 68 : = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
4761 : build_vector_from_val (atype, one),
4762 : build_vector_from_val (atype, zero));
4763 68 : gassign *new_stmt
4764 68 : = gimple_build_assign (make_ssa_name (atype),
4765 : vec_oprnd0);
4766 68 : vect_finish_stmt_generation (vinfo, stmt_info,
4767 : new_stmt, gsi);
4768 68 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4769 : }
4770 : else
4771 : {
4772 : /* The mask type has more elements than simdlen. */
4773 :
4774 : /* FORNOW */
4775 0 : gcc_unreachable ();
4776 : }
4777 : }
4778 : }
4779 7 : else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4780 : {
4781 7 : atype = bestn->simdclone->args[i].vector_type;
4782 7 : poly_uint64 atype_subparts
4783 7 : = exact_div (bestn->simdclone->simdlen,
4784 : bestn->simdclone->args[i].linear_step);
4785 7 : o = bestn->simdclone->args[i].linear_step;
4786 15 : for (m = j * o; m < (j + 1) * o; m++)
4787 : {
4788 8 : if (m == 0)
4789 7 : vec_oprnds_i[i] = 0;
4790 8 : if (maybe_lt (atype_subparts,
4791 8 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4792 : {
4793 : /* The mask argument has fewer elements than the
4794 : input vector. */
4795 : /* FORNOW */
4796 0 : gcc_unreachable ();
4797 : }
4798 8 : else if (known_eq (atype_subparts,
4799 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4800 : {
4801 8 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4802 8 : if (loop_vinfo
4803 8 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4804 : {
4805 3 : vec_loop_masks *loop_masks
4806 : = &LOOP_VINFO_MASKS (loop_vinfo);
4807 3 : tree loop_mask
4808 3 : = vect_get_loop_mask (loop_vinfo, gsi,
4809 : loop_masks, ncopies_in,
4810 : vectype, m);
4811 3 : vec_oprnd0
4812 3 : = prepare_vec_mask (loop_vinfo,
4813 3 : TREE_TYPE (loop_mask),
4814 : loop_mask, vec_oprnd0,
4815 : gsi);
4816 : }
4817 : /* The vector mask argument matches the input
4818 : in the number of lanes, but not necessarily
4819 : in the mode. */
4820 8 : tree st = lang_hooks.types.type_for_mode
4821 8 : (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
4822 8 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
4823 : vec_oprnd0);
4824 8 : gassign *new_stmt
4825 8 : = gimple_build_assign (make_ssa_name (st),
4826 : vec_oprnd0);
4827 8 : vect_finish_stmt_generation (vinfo, stmt_info,
4828 : new_stmt, gsi);
4829 8 : if (!types_compatible_p (atype, st))
4830 : {
4831 8 : new_stmt
4832 8 : = gimple_build_assign (make_ssa_name (atype),
4833 : NOP_EXPR,
4834 : gimple_assign_lhs
4835 : (new_stmt));
4836 8 : vect_finish_stmt_generation (vinfo, stmt_info,
4837 : new_stmt, gsi);
4838 : }
4839 8 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4840 : }
4841 : else
4842 : {
4843 : /* The mask argument has more elements than the
4844 : input vector. */
4845 : /* FORNOW */
4846 0 : gcc_unreachable ();
4847 : }
4848 : }
4849 : }
4850 : else
4851 0 : gcc_unreachable ();
4852 : break;
4853 102 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4854 102 : vargs.safe_push (op);
4855 102 : break;
4856 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4857 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4858 121 : if (j == 0)
4859 : {
4860 118 : gimple_seq stmts;
4861 118 : arginfo[i].op
4862 118 : = force_gimple_operand (unshare_expr (arginfo[i].op),
4863 : &stmts, true, NULL_TREE);
4864 118 : if (stmts != NULL)
4865 : {
4866 0 : basic_block new_bb;
4867 0 : edge pe = loop_preheader_edge (loop);
4868 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4869 0 : gcc_assert (!new_bb);
4870 : }
4871 118 : if (arginfo[i].simd_lane_linear)
4872 : {
4873 6 : vargs.safe_push (arginfo[i].op);
4874 6 : break;
4875 : }
4876 112 : tree phi_res = copy_ssa_name (op);
4877 112 : gphi *new_phi = create_phi_node (phi_res, loop->header);
4878 112 : add_phi_arg (new_phi, arginfo[i].op,
4879 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
4880 112 : enum tree_code code
4881 196 : = POINTER_TYPE_P (TREE_TYPE (op))
4882 112 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4883 196 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4884 196 : ? sizetype : TREE_TYPE (op);
4885 112 : poly_widest_int cst
4886 112 : = wi::mul (bestn->simdclone->args[i].linear_step,
4887 112 : ncopies * nunits);
4888 112 : tree tcst = wide_int_to_tree (type, cst);
4889 112 : tree phi_arg = copy_ssa_name (op);
4890 112 : gassign *new_stmt
4891 112 : = gimple_build_assign (phi_arg, code, phi_res, tcst);
4892 112 : gimple_stmt_iterator si = gsi_after_labels (loop->header);
4893 112 : gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4894 112 : add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4895 : UNKNOWN_LOCATION);
4896 112 : arginfo[i].op = phi_res;
4897 112 : vargs.safe_push (phi_res);
4898 112 : }
4899 : else
4900 : {
4901 3 : enum tree_code code
4902 6 : = POINTER_TYPE_P (TREE_TYPE (op))
4903 3 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4904 6 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4905 6 : ? sizetype : TREE_TYPE (op);
4906 3 : poly_widest_int cst
4907 3 : = wi::mul (bestn->simdclone->args[i].linear_step,
4908 3 : j * nunits);
4909 3 : tree tcst = wide_int_to_tree (type, cst);
4910 3 : new_temp = make_ssa_name (TREE_TYPE (op));
4911 3 : gassign *new_stmt
4912 6 : = gimple_build_assign (new_temp, code,
4913 3 : arginfo[i].op, tcst);
4914 3 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4915 3 : vargs.safe_push (new_temp);
4916 3 : }
4917 : break;
4918 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4919 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4920 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4921 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4922 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4923 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4924 0 : default:
4925 0 : gcc_unreachable ();
4926 : }
4927 : }
4928 :
4929 520 : if (masked_call_offset == 0
4930 449 : && bestn->simdclone->inbranch
4931 37 : && bestn->simdclone->nargs > nargs)
4932 : {
4933 37 : unsigned long m, o;
4934 37 : size_t mask_i = bestn->simdclone->nargs - 1;
4935 37 : tree mask;
4936 37 : gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
4937 : SIMD_CLONE_ARG_TYPE_MASK);
4938 :
4939 37 : tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
4940 37 : tree mask_vectype;
4941 37 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4942 : {
4943 2 : callee_nelements = exact_div (bestn->simdclone->simdlen,
4944 : bestn->simdclone->args[i].linear_step);
4945 2 : mask_vectype = get_related_vectype_for_scalar_type
4946 2 : (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
4947 : }
4948 : else
4949 : {
4950 35 : mask_vectype = mask_argtype;
4951 35 : callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
4952 : }
4953 37 : o = vector_unroll_factor (nunits, callee_nelements);
4954 74 : for (m = j * o; m < (j + 1) * o; m++)
4955 : {
4956 37 : if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4957 : {
4958 1 : vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
4959 1 : mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
4960 : ncopies * o, mask_vectype, m);
4961 : }
4962 36 : else if (bestn->simdclone->mask_mode == VOIDmode)
4963 : {
4964 : /* All lanes are active: pass what the VEC_COND_EXPR below
4965 : would select. An all-ones floating-point MASK_ARGTYPE
4966 : is not a valid condition. */
4967 35 : vargs.safe_push (build_one_cst (mask_argtype));
4968 35 : continue;
4969 : }
4970 : else
4971 1 : mask = vect_build_all_ones_mask (vinfo, stmt_info,
4972 : mask_argtype);
4973 :
4974 2 : gassign *new_stmt;
4975 2 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4976 : {
4977 : /* This means we are dealing with integer mask modes.
4978 : First convert to an integer type with the same size as
4979 : the current vector type. */
4980 2 : unsigned HOST_WIDE_INT intermediate_size
4981 2 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
4982 2 : tree mid_int_type =
4983 2 : build_nonstandard_integer_type (intermediate_size, 1);
4984 2 : mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
4985 2 : new_stmt
4986 2 : = gimple_build_assign (make_ssa_name (mid_int_type),
4987 : mask);
4988 2 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4989 : /* Then zero-extend to the mask mode. */
4990 2 : mask = fold_build1 (NOP_EXPR, mask_argtype,
4991 : gimple_get_lhs (new_stmt));
4992 : }
4993 0 : else if (bestn->simdclone->mask_mode == VOIDmode)
4994 0 : mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
4995 : build_one_cst (mask_argtype),
4996 : build_zero_cst (mask_argtype));
4997 : else
4998 0 : gcc_unreachable ();
4999 :
5000 2 : new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
5001 : mask);
5002 2 : vect_finish_stmt_generation (vinfo, stmt_info,
5003 : new_stmt, gsi);
5004 2 : mask = gimple_assign_lhs (new_stmt);
5005 2 : vargs.safe_push (mask);
5006 : }
5007 : }
5008 :
5009 520 : gcall *new_call = gimple_build_call_vec (fndecl, vargs);
5010 520 : if (vec_dest)
5011 : {
5012 514 : gcc_assert (ratype
5013 : || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
5014 514 : if (ratype)
5015 20 : new_temp = create_tmp_var (ratype);
5016 494 : else if (useless_type_conversion_p (vectype, rtype))
5017 438 : new_temp = make_ssa_name (vec_dest, new_call);
5018 : else
5019 56 : new_temp = make_ssa_name (rtype, new_call);
5020 514 : gimple_call_set_lhs (new_call, new_temp);
5021 : }
5022 520 : vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
5023 520 : gimple *new_stmt = new_call;
5024 :
5025 520 : if (vec_dest)
5026 : {
5027 514 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
5028 : {
5029 20 : unsigned int k, l;
5030 40 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
5031 40 : poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
5032 20 : k = vector_unroll_factor (nunits,
5033 : TYPE_VECTOR_SUBPARTS (vectype));
5034 20 : gcc_assert ((k & (k - 1)) == 0);
5035 72 : for (l = 0; l < k; l++)
5036 : {
5037 52 : tree t;
5038 52 : if (ratype)
5039 : {
5040 44 : t = build_fold_addr_expr (new_temp);
5041 44 : t = build2 (MEM_REF, vectype, t,
5042 44 : build_int_cst (TREE_TYPE (t), l * bytes));
5043 : }
5044 : else
5045 8 : t = build3 (BIT_FIELD_REF, vectype, new_temp,
5046 8 : bitsize_int (prec), bitsize_int (l * prec));
5047 52 : new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
5048 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5049 :
5050 52 : SLP_TREE_VEC_DEFS (slp_node)
5051 52 : .quick_push (gimple_assign_lhs (new_stmt));
5052 : }
5053 :
5054 20 : if (ratype)
5055 16 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5056 20 : continue;
5057 20 : }
5058 494 : else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
5059 : {
5060 56 : unsigned int k;
5061 56 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
5062 56 : TYPE_VECTOR_SUBPARTS (rtype), &k))
5063 0 : gcc_unreachable ();
5064 56 : gcc_assert ((k & (k - 1)) == 0);
5065 56 : if (!ret_ctor_elts)
5066 18 : vec_alloc (ret_ctor_elts, k);
5067 56 : if (ratype)
5068 : {
5069 4 : unsigned int m, o;
5070 4 : o = vector_unroll_factor (nunits,
5071 : TYPE_VECTOR_SUBPARTS (rtype));
5072 12 : for (m = 0; m < o; m++)
5073 : {
5074 8 : tree tem = build4 (ARRAY_REF, rtype, new_temp,
5075 8 : size_int (m), NULL_TREE, NULL_TREE);
5076 8 : new_stmt = gimple_build_assign (make_ssa_name (rtype),
5077 : tem);
5078 8 : vect_finish_stmt_generation (vinfo, stmt_info,
5079 : new_stmt, gsi);
5080 8 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
5081 : gimple_assign_lhs (new_stmt));
5082 : }
5083 4 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5084 : }
5085 : else
5086 52 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
5087 56 : if (ret_ctor_elts->length () != k)
5088 38 : continue;
5089 18 : vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
5090 18 : new_stmt
5091 18 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5092 18 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5093 18 : ret_ctor_elts = NULL;
5094 :
5095 18 : SLP_TREE_VEC_DEFS (slp_node)
5096 18 : .quick_push (gimple_assign_lhs (new_stmt));
5097 18 : continue;
5098 18 : }
5099 438 : else if (ratype)
5100 : {
5101 0 : tree t = build_fold_addr_expr (new_temp);
5102 0 : t = build2 (MEM_REF, vectype, t,
5103 0 : build_int_cst (TREE_TYPE (t), 0));
5104 0 : new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
5105 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5106 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5107 : }
5108 438 : else if (!useless_type_conversion_p (vectype, rtype))
5109 : {
5110 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
5111 0 : new_stmt
5112 0 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5113 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5114 : }
5115 : }
5116 :
5117 444 : if (gimple_get_lhs (new_stmt))
5118 438 : SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
5119 : }
5120 :
5121 1215 : for (i = 0; i < nargs; ++i)
5122 : {
5123 833 : vec<tree> oprndsi = vec_oprnds[i];
5124 833 : oprndsi.release ();
5125 : }
5126 382 : vargs.release ();
5127 :
5128 : /* Mark the clone as no longer being a candidate for GC. */
5129 382 : bestn->gc_candidate = false;
5130 :
5131 382 : return true;
5132 1492 : }
5133 :
5134 :
5135 : /* Function vect_gen_widened_results_half
5136 :
5137 : Create a vector stmt whose code, type, number of arguments, and result
5138 : variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5139 : VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5140 : In the case that CODE is a CALL_EXPR, this means that a call to DECL
5141 : needs to be created (DECL is a function-decl of a target-builtin).
5142 : STMT_INFO is the original scalar stmt that we are vectorizing. */
5143 :
5144 : static gimple *
5145 31971 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
5146 : tree vec_oprnd0, tree vec_oprnd1, int op_type,
5147 : tree vec_dest, gimple_stmt_iterator *gsi,
5148 : stmt_vec_info stmt_info)
5149 : {
5150 31971 : gimple *new_stmt;
5151 31971 : tree new_temp;
5152 :
5153 : /* Generate half of the widened result: */
5154 31971 : if (op_type != binary_op)
5155 30835 : vec_oprnd1 = NULL;
5156 31971 : new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
5157 31971 : new_temp = make_ssa_name (vec_dest, new_stmt);
5158 31971 : gimple_set_lhs (new_stmt, new_temp);
5159 31971 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5160 :
5161 31971 : return new_stmt;
5162 : }
5163 :
5164 :
5165 : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5166 : For multi-step conversions store the resulting vectors and call the function
5167 : recursively. When NARROW_SRC_P is true, there's still a conversion after
5168 : narrowing, don't store the vectors in the SLP_NODE or in vector info of
5169 : the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5170 :
5171 : static void
5172 12440 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
5173 : int multi_step_cvt,
5174 : stmt_vec_info stmt_info,
5175 : vec<tree> &vec_dsts,
5176 : gimple_stmt_iterator *gsi,
5177 : slp_tree slp_node, code_helper code,
5178 : bool narrow_src_p)
5179 : {
5180 12440 : unsigned int i;
5181 12440 : tree vop0, vop1, new_tmp, vec_dest;
5182 :
5183 12440 : vec_dest = vec_dsts.pop ();
5184 :
5185 29444 : for (i = 0; i < vec_oprnds->length (); i += 2)
5186 : {
5187 : /* Create demotion operation. */
5188 17004 : vop0 = (*vec_oprnds)[i];
5189 17004 : vop1 = (*vec_oprnds)[i + 1];
5190 17004 : gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
5191 17004 : new_tmp = make_ssa_name (vec_dest, new_stmt);
5192 17004 : gimple_set_lhs (new_stmt, new_tmp);
5193 17004 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5194 17004 : if (multi_step_cvt || narrow_src_p)
5195 : /* Store the resulting vector for next recursive call,
5196 : or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5197 7010 : (*vec_oprnds)[i/2] = new_tmp;
5198 : else
5199 : {
5200 : /* This is the last step of the conversion sequence. Store the
5201 : vectors in SLP_NODE. */
5202 9994 : slp_node->push_vec_def (new_stmt);
5203 : }
5204 : }
5205 :
5206 : /* For multi-step demotion operations we first generate demotion operations
5207 : from the source type to the intermediate types, and then combine the
5208 : results (stored in VEC_OPRNDS) in demotion operation to the destination
5209 : type. */
5210 12440 : if (multi_step_cvt)
5211 : {
5212 : /* At each level of recursion we have half of the operands we had at the
5213 : previous level. */
5214 3119 : vec_oprnds->truncate ((i+1)/2);
5215 3119 : vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
5216 : multi_step_cvt - 1,
5217 : stmt_info, vec_dsts, gsi,
5218 3119 : slp_node, VEC_PACK_TRUNC_EXPR,
5219 : narrow_src_p);
5220 : }
5221 :
5222 12440 : vec_dsts.quick_push (vec_dest);
5223 12440 : }
5224 :
5225 :
5226 : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5227 : and VEC_OPRNDS1, for a binary operation associated with scalar statement
5228 : STMT_INFO. For multi-step conversions store the resulting vectors and
5229 : call the function recursively. When NUM_VECTORS is not -1U then only
5230 : NUM_VECTORS will be produced. */
5231 :
5232 : static void
5233 11663 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
5234 : vec<tree> *vec_oprnds0,
5235 : vec<tree> *vec_oprnds1,
5236 : stmt_vec_info stmt_info, tree vec_dest,
5237 : gimple_stmt_iterator *gsi,
5238 : code_helper ch1,
5239 : code_helper ch2, int op_type,
5240 : unsigned num_vectors)
5241 : {
5242 11663 : int i;
5243 11663 : tree vop0, vop1, new_tmp1, new_tmp2;
5244 11663 : gimple *new_stmt1, *new_stmt2;
5245 11663 : vec<tree> vec_tmp = vNULL;
5246 :
5247 11663 : vec_tmp.create (num_vectors != -1U
5248 1830 : ? num_vectors : 2 * vec_oprnds0->length ());
5249 27654 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5250 : {
5251 15991 : if (op_type == binary_op)
5252 568 : vop1 = (*vec_oprnds1)[i];
5253 : else
5254 : vop1 = NULL_TREE;
5255 :
5256 : /* Generate the two halves of promotion operation. */
5257 15991 : new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
5258 : op_type, vec_dest, gsi,
5259 : stmt_info);
5260 15991 : new_tmp1 = gimple_get_lhs (new_stmt1);
5261 15991 : vec_tmp.quick_push (new_tmp1);
5262 :
5263 15991 : if (vec_tmp.space (1))
5264 : {
5265 15980 : new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
5266 : op_type, vec_dest, gsi,
5267 : stmt_info);
5268 15980 : new_tmp2 = gimple_get_lhs (new_stmt2);
5269 15980 : vec_tmp.quick_push (new_tmp2);
5270 : }
5271 :
5272 15991 : if (!vec_tmp.space (1))
5273 : break;
5274 : }
5275 :
5276 11663 : vec_oprnds0->release ();
5277 11663 : *vec_oprnds0 = vec_tmp;
5278 11663 : }
5279 :
5280 : /* Create vectorized promotion stmts for widening stmts using only half the
5281 : potential vector size for input. */
5282 : static void
5283 44 : vect_create_half_widening_stmts (vec_info *vinfo,
5284 : vec<tree> *vec_oprnds0,
5285 : vec<tree> *vec_oprnds1,
5286 : stmt_vec_info stmt_info, tree vec_dest,
5287 : gimple_stmt_iterator *gsi,
5288 : code_helper code1,
5289 : int op_type)
5290 : {
5291 44 : int i;
5292 44 : tree vop0, vop1;
5293 44 : gimple *new_stmt1;
5294 44 : gimple *new_stmt2;
5295 44 : gimple *new_stmt3;
5296 44 : vec<tree> vec_tmp = vNULL;
5297 :
5298 44 : vec_tmp.create (vec_oprnds0->length ());
5299 132 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5300 : {
5301 44 : tree new_tmp1, new_tmp2, new_tmp3, out_type;
5302 :
5303 44 : gcc_assert (op_type == binary_op);
5304 44 : vop1 = (*vec_oprnds1)[i];
5305 :
5306 : /* Widen the first vector input. */
5307 44 : out_type = TREE_TYPE (vec_dest);
5308 44 : new_tmp1 = make_ssa_name (out_type);
5309 44 : new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
5310 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
5311 44 : if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
5312 : {
5313 : /* Widen the second vector input. */
5314 44 : new_tmp2 = make_ssa_name (out_type);
5315 44 : new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
5316 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
5317 : /* Perform the operation. With both vector inputs widened. */
5318 44 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
5319 : }
5320 : else
5321 : {
5322 : /* Perform the operation. With the single vector input widened. */
5323 0 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
5324 : }
5325 :
5326 44 : new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
5327 44 : gimple_assign_set_lhs (new_stmt3, new_tmp3);
5328 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
5329 :
5330 : /* Store the results for the next step. */
5331 44 : vec_tmp.quick_push (new_tmp3);
5332 : }
5333 :
5334 44 : vec_oprnds0->release ();
5335 44 : *vec_oprnds0 = vec_tmp;
5336 44 : }
5337 :
5338 :
5339 : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5340 : If COST_VEC is passed, calculate costs but don't change anything,
5341 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5342 : it, and insert it at GSI.
5343 : Return true if STMT_INFO is vectorizable in this way. */
5344 :
5345 : static bool
5346 2689066 : vectorizable_conversion (vec_info *vinfo,
5347 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5348 : slp_tree slp_node,
5349 : stmt_vector_for_cost *cost_vec)
5350 : {
5351 2689066 : tree vec_dest, cvt_op = NULL_TREE;
5352 2689066 : tree scalar_dest;
5353 2689066 : tree_code tc1;
5354 2689066 : code_helper code, code1, code2;
5355 2689066 : code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
5356 2689066 : tree new_temp;
5357 2689066 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5358 2689066 : poly_uint64 nunits_in;
5359 2689066 : poly_uint64 nunits_out;
5360 2689066 : tree vectype_out, vectype_in;
5361 2689066 : int i;
5362 2689066 : tree lhs_type, rhs_type;
5363 : /* For conversions between floating point and integer, there're 2 NARROW
5364 : cases. NARROW_SRC is for FLOAT_EXPR, means
5365 : integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5366 : This is safe when the range of the source integer can fit into the lower
5367 : precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5368 : floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5369 : For other conversions, when there's narrowing, NARROW_DST is used as
5370 : default. */
5371 2689066 : enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
5372 2689066 : vec<tree> vec_oprnds0 = vNULL;
5373 2689066 : vec<tree> vec_oprnds1 = vNULL;
5374 2689066 : tree vop0;
5375 2689066 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5376 2689066 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5377 2689066 : int multi_step_cvt = 0;
5378 2689066 : vec<tree> interm_types = vNULL;
5379 2689066 : tree intermediate_type, cvt_type = NULL_TREE;
5380 2689066 : int op_type;
5381 2689066 : unsigned short fltsz;
5382 :
5383 : /* Is STMT a vectorizable conversion? */
5384 :
5385 2689066 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5386 : return false;
5387 :
5388 2689066 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5389 251571 : && cost_vec)
5390 : return false;
5391 :
5392 2437495 : gimple* stmt = stmt_info->stmt;
5393 2437495 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
5394 : return false;
5395 :
5396 2375585 : if (gimple_get_lhs (stmt) == NULL_TREE
5397 2375585 : || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5398 : return false;
5399 :
5400 1538133 : if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5401 : return false;
5402 :
5403 1538133 : if (is_gimple_assign (stmt))
5404 : {
5405 1524618 : code = gimple_assign_rhs_code (stmt);
5406 1524618 : op_type = TREE_CODE_LENGTH ((tree_code) code);
5407 : }
5408 13515 : else if (gimple_call_internal_p (stmt))
5409 : {
5410 7827 : code = gimple_call_internal_fn (stmt);
5411 7827 : op_type = gimple_call_num_args (stmt);
5412 : }
5413 : else
5414 : return false;
5415 :
5416 1532445 : bool widen_arith = (code == WIDEN_MULT_EXPR
5417 1529819 : || code == WIDEN_LSHIFT_EXPR
5418 3062264 : || widening_fn_p (code));
5419 :
5420 1529819 : if (!widen_arith
5421 1529819 : && !CONVERT_EXPR_CODE_P (code)
5422 : && code != FIX_TRUNC_EXPR
5423 : && code != FLOAT_EXPR)
5424 : return false;
5425 :
5426 : /* Check types of lhs and rhs. */
5427 202041 : scalar_dest = gimple_get_lhs (stmt);
5428 202041 : lhs_type = TREE_TYPE (scalar_dest);
5429 202041 : vectype_out = SLP_TREE_VECTYPE (slp_node);
5430 :
5431 : /* Check the operands of the operation. */
5432 202041 : slp_tree slp_op0, slp_op1 = NULL;
5433 202041 : if (!vect_is_simple_use (vinfo, slp_node,
5434 : 0, &slp_op0, &dt[0], &vectype_in))
5435 : {
5436 0 : if (dump_enabled_p ())
5437 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5438 : "use not simple.\n");
5439 : return false;
5440 : }
5441 :
5442 202041 : rhs_type = TREE_TYPE (gimple_arg (stmt, 0));
5443 202041 : if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
5444 384107 : && !((INTEGRAL_TYPE_P (lhs_type)
5445 166983 : && INTEGRAL_TYPE_P (rhs_type))
5446 : || (SCALAR_FLOAT_TYPE_P (lhs_type)
5447 10165 : && SCALAR_FLOAT_TYPE_P (rhs_type))))
5448 : return false;
5449 :
5450 197123 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5451 176559 : && INTEGRAL_TYPE_P (lhs_type)
5452 341516 : && !type_has_mode_precision_p (lhs_type))
5453 : {
5454 581 : if (dump_enabled_p ())
5455 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5456 : "type conversion to bit-precision unsupported\n");
5457 : return false;
5458 : }
5459 :
5460 : /* _BitInt values are not sign-/zero-extended to mode precision. */
5461 196542 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5462 175978 : && TREE_CODE (rhs_type) == BITINT_TYPE
5463 196715 : && !type_has_mode_precision_p (rhs_type))
5464 : {
5465 16 : if (dump_enabled_p ())
5466 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5467 : "type conversion from _BitInt unsupported\n");
5468 : return false;
5469 : }
5470 :
5471 196526 : if (op_type == binary_op)
5472 : {
5473 2626 : gcc_assert (code == WIDEN_MULT_EXPR
5474 : || code == WIDEN_LSHIFT_EXPR
5475 : || widening_fn_p (code));
5476 :
5477 2626 : tree vectype1_in;
5478 2626 : if (!vect_is_simple_use (vinfo, slp_node, 1,
5479 : &slp_op1, &dt[1], &vectype1_in))
5480 : {
5481 0 : if (dump_enabled_p ())
5482 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5483 : "use not simple.\n");
5484 0 : return false;
5485 : }
5486 : /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5487 : OP1. */
5488 2626 : if (!vectype_in)
5489 171 : vectype_in = vectype1_in;
5490 : }
5491 :
5492 : /* If slp_op0 is an external or constant def, infer the vector type
5493 : from the scalar type. */
5494 196526 : if (!cost_vec)
5495 23709 : gcc_assert (vectype_in);
5496 196526 : if (!vectype_in)
5497 25289 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
5498 196526 : if (!vectype_in)
5499 : {
5500 280 : if (dump_enabled_p ())
5501 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5502 : "no vectype for scalar type %T\n", rhs_type);
5503 :
5504 : return false;
5505 : }
5506 :
5507 392492 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
5508 196246 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
5509 : {
5510 309 : if (dump_enabled_p ())
5511 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5512 : "can't convert between boolean and non "
5513 : "boolean vectors %T\n", rhs_type);
5514 :
5515 : return false;
5516 : }
5517 :
5518 195937 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
5519 195937 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5520 195937 : if (known_eq (nunits_out, nunits_in))
5521 93980 : if (widen_arith)
5522 : modifier = WIDEN;
5523 : else
5524 195937 : modifier = NONE;
5525 101957 : else if (multiple_p (nunits_out, nunits_in))
5526 : modifier = NARROW_DST;
5527 : else
5528 : {
5529 56827 : gcc_checking_assert (multiple_p (nunits_in, nunits_out));
5530 : modifier = WIDEN;
5531 : }
5532 :
5533 195937 : bool found_mode = false;
5534 195937 : scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
5535 195937 : scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
5536 195937 : opt_scalar_mode rhs_mode_iter;
5537 195937 : auto_vec<std::pair<tree, tree_code>, 2> converts;
5538 195937 : bool evenodd_ok = false;
5539 :
5540 : /* Supportable by target? */
5541 195937 : switch (modifier)
5542 : {
5543 93619 : case NONE:
5544 93619 : if (code != FIX_TRUNC_EXPR
5545 : && code != FLOAT_EXPR
5546 : && !CONVERT_EXPR_CODE_P (code))
5547 : return false;
5548 93619 : gcc_assert (code.is_tree_code ());
5549 93619 : if (supportable_indirect_convert_operation (code,
5550 : vectype_out, vectype_in,
5551 : converts, slp_op0))
5552 : {
5553 86700 : gcc_assert (converts.length () <= 2);
5554 86700 : if (converts.length () == 1)
5555 : {
5556 86626 : code1 = converts[0].second;
5557 86626 : if (CONVERT_EXPR_CODE_P (code)
5558 162701 : && tree_nop_conversion_p (TREE_TYPE (vectype_out),
5559 76075 : TREE_TYPE (vectype_in)))
5560 : /* NOP conversions are handled by vectorizable_assignment. */
5561 : return false;
5562 : }
5563 : else
5564 : {
5565 74 : cvt_type = NULL_TREE;
5566 74 : multi_step_cvt = converts.length () - 1;
5567 74 : codecvt1 = converts[0].second;
5568 74 : code1 = converts[1].second;
5569 74 : interm_types.safe_push (converts[0].first);
5570 : }
5571 : break;
5572 : }
5573 :
5574 : /* FALLTHRU */
5575 6919 : unsupported:
5576 19613 : if (dump_enabled_p ())
5577 618 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5578 : "conversion not supported by target.\n");
5579 : return false;
5580 :
5581 57188 : case WIDEN:
5582 57188 : if (known_eq (nunits_in, nunits_out))
5583 : {
5584 722 : if (!(code.is_tree_code ()
5585 361 : && supportable_half_widening_operation ((tree_code) code,
5586 : vectype_out, vectype_in,
5587 : &tc1)))
5588 119 : goto unsupported;
5589 242 : code1 = tc1;
5590 242 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5591 : break;
5592 : }
5593 : /* Elements in a vector can only be reordered if used in a reduction
5594 : operation only. */
5595 56827 : if (code == WIDEN_MULT_EXPR
5596 2265 : && loop_vinfo
5597 2124 : && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
5598 : /* For a SLP reduction we cannot swizzle lanes, detecting a
5599 : reduction chain isn't possible here. */
5600 58929 : && SLP_TREE_LANES (slp_node) == 1)
5601 : {
5602 : /* ??? There is no way to look for SLP uses, so work on
5603 : the stmt and what the stmt-based cycle detection gives us. */
5604 4008 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5605 2004 : stmt_vec_info use_stmt_info
5606 2004 : = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
5607 2004 : if (use_stmt_info
5608 1859 : && STMT_VINFO_REDUC_DEF (use_stmt_info))
5609 56827 : evenodd_ok = true;
5610 : }
5611 56827 : if (supportable_widening_operation (code, vectype_out, vectype_in,
5612 : evenodd_ok, &code1,
5613 : &code2, &multi_step_cvt,
5614 : &interm_types))
5615 : {
5616 : /* Binary widening operation can only be supported directly by the
5617 : architecture. */
5618 50398 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5619 : break;
5620 : }
5621 :
5622 6429 : if (code != FLOAT_EXPR
5623 6811 : || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
5624 6238 : goto unsupported;
5625 :
5626 191 : fltsz = GET_MODE_SIZE (lhs_mode);
5627 278 : FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
5628 : {
5629 278 : rhs_mode = rhs_mode_iter.require ();
5630 556 : if (GET_MODE_SIZE (rhs_mode) > fltsz)
5631 : break;
5632 :
5633 278 : cvt_type
5634 278 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5635 278 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5636 278 : if (cvt_type == NULL_TREE)
5637 0 : goto unsupported;
5638 :
5639 556 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5640 : {
5641 86 : gcc_assert (code.is_tree_code ());
5642 86 : if (!supportable_convert_operation ((tree_code) code, vectype_out,
5643 : cvt_type))
5644 22 : goto unsupported;
5645 64 : codecvt1 = code;
5646 : }
5647 192 : else if (!supportable_widening_operation (code, vectype_out,
5648 : cvt_type, evenodd_ok,
5649 : &codecvt1,
5650 : &codecvt2, &multi_step_cvt,
5651 : &interm_types))
5652 87 : continue;
5653 : else
5654 105 : gcc_assert (multi_step_cvt == 0);
5655 :
5656 169 : if (supportable_widening_operation (NOP_EXPR, cvt_type,
5657 : vectype_in, evenodd_ok, &code1,
5658 : &code2, &multi_step_cvt,
5659 : &interm_types))
5660 : {
5661 : found_mode = true;
5662 : break;
5663 : }
5664 : }
5665 :
5666 169 : if (!found_mode)
5667 0 : goto unsupported;
5668 :
5669 338 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5670 64 : codecvt2 = ERROR_MARK;
5671 : else
5672 : {
5673 105 : multi_step_cvt++;
5674 105 : interm_types.safe_push (cvt_type);
5675 105 : cvt_type = NULL_TREE;
5676 : }
5677 : break;
5678 :
5679 45130 : case NARROW_DST:
5680 45130 : gcc_assert (op_type == unary_op);
5681 45130 : if (supportable_narrowing_operation (code, vectype_out, vectype_in,
5682 : &code1, &multi_step_cvt,
5683 : &interm_types))
5684 : break;
5685 :
5686 19347 : if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
5687 986 : goto unsupported;
5688 :
5689 5463 : if (code == FIX_TRUNC_EXPR)
5690 : {
5691 107 : cvt_type
5692 107 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5693 107 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5694 107 : if (cvt_type == NULL_TREE)
5695 0 : goto unsupported;
5696 107 : if (supportable_convert_operation ((tree_code) code, cvt_type,
5697 : vectype_in))
5698 105 : codecvt1 = code;
5699 : else
5700 2 : goto unsupported;
5701 105 : if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
5702 : &code1, &multi_step_cvt,
5703 : &interm_types))
5704 : break;
5705 : }
5706 : /* If slp_op0 can be represented with low precision integer,
5707 : truncate it to cvt_type and the do FLOAT_EXPR. */
5708 5356 : else if (code == FLOAT_EXPR)
5709 : {
5710 792 : if (cost_vec)
5711 : {
5712 784 : wide_int op_min_value, op_max_value;
5713 784 : tree def;
5714 :
5715 : /* ??? Merge ranges in case of more than one lane. */
5716 784 : if (SLP_TREE_LANES (slp_op0) != 1
5717 133 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
5718 917 : || !vect_get_range_info (def, &op_min_value, &op_max_value))
5719 755 : goto unsupported;
5720 :
5721 29 : if ((wi::min_precision (op_max_value, SIGNED)
5722 29 : > GET_MODE_BITSIZE (lhs_mode))
5723 29 : || (wi::min_precision (op_min_value, SIGNED)
5724 27 : > GET_MODE_BITSIZE (lhs_mode)))
5725 2 : goto unsupported;
5726 784 : }
5727 :
5728 35 : cvt_type
5729 35 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
5730 35 : cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
5731 35 : if (cvt_type == NULL_TREE)
5732 0 : goto unsupported;
5733 35 : if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
5734 : &code1, &multi_step_cvt,
5735 : &interm_types))
5736 2 : goto unsupported;
5737 33 : if (supportable_convert_operation ((tree_code) code, vectype_out,
5738 : cvt_type))
5739 : {
5740 33 : codecvt1 = code;
5741 33 : modifier = NARROW_SRC;
5742 33 : break;
5743 : }
5744 : }
5745 :
5746 4568 : goto unsupported;
5747 :
5748 : default:
5749 : gcc_unreachable ();
5750 : }
5751 :
5752 111260 : if (modifier == WIDEN
5753 111260 : && loop_vinfo
5754 49240 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
5755 132492 : && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
5756 21214 : || widening_evenodd_fn_p (code1)))
5757 : {
5758 18 : if (dump_enabled_p ())
5759 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5760 : "can't use a fully-masked loop because"
5761 : " widening operation on even/odd elements"
5762 : " mixes up lanes.\n");
5763 18 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
5764 : }
5765 :
5766 111260 : if (cost_vec) /* transformation not required. */
5767 : {
5768 87551 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
5769 87551 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
5770 : {
5771 0 : if (dump_enabled_p ())
5772 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5773 : "incompatible vector types for invariants\n");
5774 : return false;
5775 : }
5776 87551 : DUMP_VECT_SCOPE ("vectorizable_conversion");
5777 87551 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
5778 87551 : if (modifier == NONE)
5779 : {
5780 17125 : SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
5781 17125 : vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
5782 : slp_node, cost_vec);
5783 : }
5784 70426 : else if (modifier == NARROW_SRC || modifier == NARROW_DST)
5785 : {
5786 29494 : SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
5787 : /* The final packing step produces one vector result per copy. */
5788 29494 : vect_model_promotion_demotion_cost (slp_node, nvectors,
5789 : multi_step_cvt, cost_vec,
5790 : widen_arith);
5791 : }
5792 : else
5793 : {
5794 40932 : SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
5795 : /* The initial unpacking step produces two vector results
5796 : per copy. MULTI_STEP_CVT is 0 for a single conversion,
5797 : so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5798 40932 : vect_model_promotion_demotion_cost (slp_node,
5799 : nvectors >> multi_step_cvt,
5800 : multi_step_cvt, cost_vec,
5801 : widen_arith);
5802 : }
5803 87551 : interm_types.release ();
5804 87551 : return true;
5805 : }
5806 :
5807 : /* Transform. */
5808 23709 : if (dump_enabled_p ())
5809 4488 : dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
5810 :
5811 : /* In case of multi-step conversion, we first generate conversion operations
5812 : to the intermediate types, and then from that types to the final one.
5813 : We create vector destinations for the intermediate type (TYPES) received
5814 : from supportable_*_operation, and store them in the correct order
5815 : for future use in vect_create_vectorized_*_stmts (). */
5816 47418 : auto_vec<tree> vec_dsts (multi_step_cvt + 1);
5817 23709 : bool widen_or_narrow_float_p
5818 23709 : = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
5819 23709 : vec_dest = vect_create_destination_var (scalar_dest,
5820 : widen_or_narrow_float_p
5821 : ? cvt_type : vectype_out);
5822 23709 : vec_dsts.quick_push (vec_dest);
5823 :
5824 23709 : if (multi_step_cvt)
5825 : {
5826 9455 : for (i = interm_types.length () - 1;
5827 9455 : interm_types.iterate (i, &intermediate_type); i--)
5828 : {
5829 4970 : vec_dest = vect_create_destination_var (scalar_dest,
5830 : intermediate_type);
5831 4970 : vec_dsts.quick_push (vec_dest);
5832 : }
5833 : }
5834 :
5835 23709 : if (cvt_type)
5836 80 : vec_dest = vect_create_destination_var (scalar_dest,
5837 : widen_or_narrow_float_p
5838 : ? vectype_out : cvt_type);
5839 :
5840 23709 : unsigned num_vectors = vect_get_num_copies (vinfo, slp_node);
5841 23709 : switch (modifier)
5842 : {
5843 4511 : case NONE:
5844 4511 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5845 : /* vec_dest is intermediate type operand when multi_step_cvt. */
5846 4511 : if (multi_step_cvt)
5847 : {
5848 21 : cvt_op = vec_dest;
5849 21 : vec_dest = vec_dsts[0];
5850 : }
5851 :
5852 9415 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5853 : {
5854 : /* Arguments are ready, create the new vector stmt. */
5855 4904 : gimple* new_stmt;
5856 4904 : if (multi_step_cvt)
5857 : {
5858 21 : gcc_assert (multi_step_cvt == 1);
5859 21 : new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
5860 21 : new_temp = make_ssa_name (cvt_op, new_stmt);
5861 21 : gimple_assign_set_lhs (new_stmt, new_temp);
5862 21 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5863 21 : vop0 = new_temp;
5864 : }
5865 4904 : if (code1 == COND_EXPR)
5866 : {
5867 1 : gcc_assert (!multi_step_cvt);
5868 1 : new_stmt
5869 2 : = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
5870 : build_minus_one_cst
5871 1 : (TREE_TYPE (vec_dest)),
5872 1 : build_zero_cst (TREE_TYPE (vec_dest)));
5873 1 : new_temp = make_ssa_name (vec_dest, new_stmt);
5874 1 : gimple_set_lhs (new_stmt, new_temp);
5875 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5876 1 : tree new_temp2 = make_ssa_name (vectype_out);
5877 1 : new_stmt = gimple_build_assign (new_temp2,
5878 : build1 (VIEW_CONVERT_EXPR,
5879 : vectype_out, new_temp));
5880 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5881 1 : slp_node->push_vec_def (new_temp2);
5882 : }
5883 : else
5884 : {
5885 4903 : new_stmt = vect_gimple_build (vec_dest, code1, vop0);
5886 4903 : new_temp = make_ssa_name (vec_dest, new_stmt);
5887 4903 : gimple_set_lhs (new_stmt, new_temp);
5888 4903 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5889 :
5890 4903 : slp_node->push_vec_def (new_stmt);
5891 : }
5892 : }
5893 : break;
5894 :
5895 9877 : case WIDEN:
5896 : /* In case the vectorization factor (VF) is bigger than the number
5897 : of elements that we can fit in a vectype (nunits), we have to
5898 : generate more than one vector stmt - i.e - we need to "unroll"
5899 : the vector stmt by a factor VF/nunits. */
5900 19754 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
5901 9877 : code != WIDEN_LSHIFT_EXPR && slp_op1, &vec_oprnds1);
5902 9877 : if (code == WIDEN_LSHIFT_EXPR)
5903 : {
5904 0 : int oprnds_size = vec_oprnds0.length ();
5905 0 : vec_oprnds1.create (oprnds_size);
5906 0 : for (i = 0; i < oprnds_size; ++i)
5907 0 : vec_oprnds1.quick_push (vect_get_slp_scalar_def (slp_op1, 0));
5908 : }
5909 : /* Arguments are ready. Create the new vector stmts. */
5910 21584 : for (i = multi_step_cvt; i >= 0; i--)
5911 : {
5912 11707 : tree this_dest = vec_dsts[i];
5913 11707 : code_helper c1 = code1, c2 = code2;
5914 11707 : if (i == 0 && codecvt2 != ERROR_MARK)
5915 : {
5916 48 : c1 = codecvt1;
5917 48 : c2 = codecvt2;
5918 : }
5919 11707 : if (known_eq (nunits_out, nunits_in))
5920 44 : vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
5921 : stmt_info, this_dest, gsi, c1,
5922 : op_type);
5923 : else
5924 : /* ??? For constant/external inputs we can end up with
5925 : excess lanes. When the number of inputs already match
5926 : the number of required outputs request half of the
5927 : lanes (gcc.dg/vect/O3-vect-pr32243.c). Low coverage
5928 : makes this likely incomplete. */
5929 13493 : vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
5930 : &vec_oprnds1, stmt_info,
5931 : this_dest, gsi,
5932 : c1, c2, op_type,
5933 : i == 0 ? num_vectors : -1u);
5934 : }
5935 :
5936 61343 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5937 : {
5938 27757 : gimple *new_stmt;
5939 27757 : if (cvt_type)
5940 : {
5941 128 : new_temp = make_ssa_name (vec_dest);
5942 128 : new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5943 128 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5944 : }
5945 : else
5946 27629 : new_stmt = SSA_NAME_DEF_STMT (vop0);
5947 :
5948 27757 : slp_node->push_vec_def (new_stmt);
5949 : }
5950 : break;
5951 :
5952 9321 : case NARROW_SRC:
5953 9321 : case NARROW_DST:
5954 : /* In case the vectorization factor (VF) is bigger than the number
5955 : of elements that we can fit in a vectype (nunits), we have to
5956 : generate more than one vector stmt - i.e - we need to "unroll"
5957 : the vector stmt by a factor VF/nunits. */
5958 9321 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5959 : /* Arguments are ready. Create the new vector stmts. */
5960 9321 : if (cvt_type && modifier == NARROW_DST)
5961 9474 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5962 : {
5963 124 : new_temp = make_ssa_name (vec_dest);
5964 124 : gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5965 124 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5966 124 : vec_oprnds0[i] = new_temp;
5967 : }
5968 :
5969 9321 : vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
5970 : multi_step_cvt,
5971 : stmt_info, vec_dsts, gsi,
5972 : slp_node, code1,
5973 : modifier == NARROW_SRC);
5974 : /* After demoting slp_op0 to cvt_type, convert it to dest. */
5975 9321 : if (cvt_type && code == FLOAT_EXPR)
5976 : {
5977 32 : for (unsigned int i = 0; i != vec_oprnds0.length() / 2; i++)
5978 : {
5979 : /* Arguments are ready, create the new vector stmt. */
5980 8 : gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
5981 8 : gimple *new_stmt
5982 8 : = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
5983 8 : new_temp = make_ssa_name (vec_dest, new_stmt);
5984 8 : gimple_set_lhs (new_stmt, new_temp);
5985 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5986 :
5987 : /* This is the last step of the conversion sequence. Store the
5988 : vectors in SLP_NODE or in vector info of the scalar statement
5989 : (or in STMT_VINFO_RELATED_STMT chain). */
5990 8 : slp_node->push_vec_def (new_stmt);
5991 : }
5992 : }
5993 : break;
5994 : }
5995 :
5996 23709 : vec_oprnds0.release ();
5997 23709 : vec_oprnds1.release ();
5998 23709 : interm_types.release ();
5999 :
6000 23709 : return true;
6001 195937 : }
6002 :
6003 : /* Return true if we can assume from the scalar form of STMT_INFO that
6004 : neither the scalar nor the vector forms will generate code. STMT_INFO
6005 : is known not to involve a data reference. */
6006 :
6007 : bool
6008 3359507 : vect_nop_conversion_p (stmt_vec_info stmt_info)
6009 : {
6010 3359507 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6011 3049605 : if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
6012 : return false;
6013 :
6014 1022919 : tree lhs = gimple_assign_lhs (stmt);
6015 1022919 : tree_code code = gimple_assign_rhs_code (stmt);
6016 1022919 : tree rhs = gimple_assign_rhs1 (stmt);
6017 :
6018 1022919 : if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
6019 : return true;
6020 :
6021 1019554 : if (CONVERT_EXPR_CODE_P (code))
6022 255436 : return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
6023 :
6024 : return false;
6025 : }
6026 :
6027 : /* Function vectorizable_assignment.
6028 :
6029 : Check if STMT_INFO performs an assignment (copy) that can be vectorized.
6030 : If COST_VEC is passed, calculate costs but don't change anything,
6031 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6032 : it, and insert it at GSI.
6033 : Return true if STMT_INFO is vectorizable in this way. */
6034 :
6035 : static bool
6036 2170961 : vectorizable_assignment (vec_info *vinfo,
6037 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6038 : slp_tree slp_node,
6039 : stmt_vector_for_cost *cost_vec)
6040 : {
6041 2170961 : tree vec_dest;
6042 2170961 : tree scalar_dest;
6043 2170961 : tree op;
6044 2170961 : tree new_temp;
6045 2170961 : enum vect_def_type dt[1] = {vect_unknown_def_type};
6046 2170961 : int i;
6047 2170961 : vec<tree> vec_oprnds = vNULL;
6048 2170961 : tree vop;
6049 2170961 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6050 2170961 : enum tree_code code;
6051 2170961 : tree vectype_in;
6052 :
6053 2170961 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6054 : return false;
6055 :
6056 2170961 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6057 251571 : && cost_vec)
6058 : return false;
6059 :
6060 : /* Is vectorizable assignment? */
6061 1919390 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6062 1842614 : if (!stmt)
6063 : return false;
6064 :
6065 1842614 : scalar_dest = gimple_assign_lhs (stmt);
6066 1842614 : if (TREE_CODE (scalar_dest) != SSA_NAME)
6067 : return false;
6068 :
6069 1006513 : if (STMT_VINFO_DATA_REF (stmt_info))
6070 : return false;
6071 :
6072 423991 : code = gimple_assign_rhs_code (stmt);
6073 423991 : if (!(gimple_assign_single_p (stmt)
6074 422337 : || code == PAREN_EXPR
6075 421065 : || CONVERT_EXPR_CODE_P (code)))
6076 : return false;
6077 :
6078 106134 : tree vectype = SLP_TREE_VECTYPE (slp_node);
6079 106134 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6080 :
6081 106134 : slp_tree slp_op;
6082 106134 : if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
6083 : &dt[0], &vectype_in))
6084 : {
6085 0 : if (dump_enabled_p ())
6086 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6087 : "use not simple.\n");
6088 : return false;
6089 : }
6090 106134 : if (!vectype_in)
6091 20104 : vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
6092 :
6093 : /* We can handle VIEW_CONVERT conversions that do not change the number
6094 : of elements or the vector size or other conversions when the component
6095 : types are nop-convertible. */
6096 106134 : if (!vectype_in
6097 110794 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
6098 93542 : || (code == VIEW_CONVERT_EXPR
6099 3020 : && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
6100 3020 : GET_MODE_SIZE (TYPE_MODE (vectype_in))))
6101 199676 : || (CONVERT_EXPR_CODE_P (code)
6102 90648 : && !tree_nop_conversion_p (TREE_TYPE (vectype),
6103 90648 : TREE_TYPE (vectype_in))))
6104 : return false;
6105 :
6106 266193 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
6107 : {
6108 27 : if (dump_enabled_p ())
6109 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6110 : "can't convert between boolean and non "
6111 0 : "boolean vectors %T\n", TREE_TYPE (op));
6112 :
6113 : return false;
6114 : }
6115 :
6116 : /* We do not handle bit-precision changes. */
6117 88855 : if ((CONVERT_EXPR_CODE_P (code)
6118 2894 : || code == VIEW_CONVERT_EXPR)
6119 87471 : && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6120 86183 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6121 86925 : || (INTEGRAL_TYPE_P (TREE_TYPE (op))
6122 78867 : && !type_has_mode_precision_p (TREE_TYPE (op))))
6123 : /* But a conversion that does not change the bit-pattern is ok. */
6124 90003 : && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6125 1148 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
6126 1148 : && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
6127 1148 : > TYPE_PRECISION (TREE_TYPE (op)))
6128 602 : && TYPE_UNSIGNED (TREE_TYPE (op)))
6129 608 : || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
6130 608 : == TYPE_PRECISION (TREE_TYPE (op))))))
6131 : {
6132 377 : if (dump_enabled_p ())
6133 42 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6134 : "type conversion to/from bit-precision "
6135 : "unsupported.\n");
6136 : return false;
6137 : }
6138 :
6139 88478 : if (cost_vec) /* transformation not required. */
6140 : {
6141 71915 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
6142 : {
6143 0 : if (dump_enabled_p ())
6144 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6145 : "incompatible vector types for invariants\n");
6146 : return false;
6147 : }
6148 71915 : SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
6149 71915 : DUMP_VECT_SCOPE ("vectorizable_assignment");
6150 71915 : if (!vect_nop_conversion_p (stmt_info))
6151 1118 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6152 71915 : return true;
6153 : }
6154 :
6155 : /* Transform. */
6156 16563 : if (dump_enabled_p ())
6157 3819 : dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
6158 :
6159 : /* Handle def. */
6160 16563 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6161 :
6162 : /* Handle use. */
6163 16563 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
6164 :
6165 : /* Arguments are ready. create the new vector stmt. */
6166 53700 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
6167 : {
6168 20574 : gassign *new_stmt;
6169 20574 : if (code == PAREN_EXPR)
6170 510 : new_stmt = gimple_build_assign (vec_dest, PAREN_EXPR, vop);
6171 : else
6172 : {
6173 20064 : if (CONVERT_EXPR_CODE_P (code)
6174 208 : || code == VIEW_CONVERT_EXPR)
6175 20004 : vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
6176 20064 : new_stmt = gimple_build_assign (vec_dest, vop);
6177 : }
6178 20574 : new_temp = make_ssa_name (vec_dest, new_stmt);
6179 20574 : gimple_assign_set_lhs (new_stmt, new_temp);
6180 20574 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6181 20574 : slp_node->push_vec_def (new_stmt);
6182 : }
6183 :
6184 16563 : vec_oprnds.release ();
6185 16563 : return true;
6186 : }
6187 :
6188 :
6189 : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6190 : either as shift by a scalar or by a vector. */
6191 :
6192 : bool
6193 317200 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
6194 : {
6195 317200 : optab optab;
6196 317200 : tree vectype;
6197 :
6198 317200 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6199 317200 : if (!vectype)
6200 : return false;
6201 :
6202 317200 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6203 317200 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6204 : return true;
6205 :
6206 278508 : optab = optab_for_tree_code (code, vectype, optab_vector);
6207 278508 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6208 : return true;
6209 :
6210 : return false;
6211 : }
6212 :
6213 :
6214 : /* Function vectorizable_shift.
6215 :
6216 : Check if STMT_INFO performs a shift operation that can be vectorized.
6217 : If COST_VEC is passed, calculate costs but don't change anything,
6218 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6219 : it, and insert it at GSI.
6220 : Return true if STMT_INFO is vectorizable in this way. */
6221 :
6222 : static bool
6223 789659 : vectorizable_shift (vec_info *vinfo,
6224 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6225 : slp_tree slp_node,
6226 : stmt_vector_for_cost *cost_vec)
6227 : {
6228 789659 : tree vec_dest;
6229 789659 : tree scalar_dest;
6230 789659 : tree op0, op1 = NULL;
6231 789659 : tree vec_oprnd1 = NULL_TREE;
6232 789659 : tree vectype;
6233 789659 : enum tree_code code;
6234 789659 : machine_mode vec_mode;
6235 789659 : tree new_temp;
6236 789659 : optab optab;
6237 789659 : int icode;
6238 789659 : machine_mode optab_op2_mode;
6239 789659 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
6240 789659 : poly_uint64 nunits_in;
6241 789659 : poly_uint64 nunits_out;
6242 789659 : tree vectype_out;
6243 789659 : tree op1_vectype;
6244 789659 : int i;
6245 789659 : vec<tree> vec_oprnds0 = vNULL;
6246 789659 : vec<tree> vec_oprnds1 = vNULL;
6247 789659 : tree vop0, vop1;
6248 789659 : unsigned int k;
6249 789659 : bool scalar_shift_arg = true;
6250 789659 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6251 789659 : bool incompatible_op1_vectype_p = false;
6252 :
6253 789659 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6254 : return false;
6255 :
6256 789659 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6257 251571 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
6258 250179 : && cost_vec)
6259 : return false;
6260 :
6261 : /* Is STMT a vectorizable binary/unary operation? */
6262 539480 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6263 463804 : if (!stmt)
6264 : return false;
6265 :
6266 463804 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
6267 : return false;
6268 :
6269 463346 : code = gimple_assign_rhs_code (stmt);
6270 :
6271 463346 : if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
6272 : || code == RROTATE_EXPR))
6273 : return false;
6274 :
6275 65101 : scalar_dest = gimple_assign_lhs (stmt);
6276 65101 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6277 65101 : if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6278 : {
6279 0 : if (dump_enabled_p ())
6280 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6281 : "bit-precision shifts not supported.\n");
6282 : return false;
6283 : }
6284 :
6285 65101 : slp_tree slp_op0;
6286 65101 : if (!vect_is_simple_use (vinfo, slp_node,
6287 : 0, &op0, &slp_op0, &dt[0], &vectype))
6288 : {
6289 0 : if (dump_enabled_p ())
6290 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6291 : "use not simple.\n");
6292 : return false;
6293 : }
6294 : /* If op0 is an external or constant def, infer the vector type
6295 : from the scalar type. */
6296 65101 : if (!vectype)
6297 14590 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
6298 65101 : if (!cost_vec)
6299 9232 : gcc_assert (vectype);
6300 65101 : if (!vectype)
6301 : {
6302 0 : if (dump_enabled_p ())
6303 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6304 : "no vectype for scalar type\n");
6305 : return false;
6306 : }
6307 :
6308 65101 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6309 65101 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6310 65101 : if (maybe_ne (nunits_out, nunits_in))
6311 : return false;
6312 :
6313 65101 : stmt_vec_info op1_def_stmt_info;
6314 65101 : slp_tree slp_op1;
6315 65101 : if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
6316 : &dt[1], &op1_vectype, &op1_def_stmt_info))
6317 : {
6318 0 : if (dump_enabled_p ())
6319 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6320 : "use not simple.\n");
6321 : return false;
6322 : }
6323 :
6324 : /* Determine whether the shift amount is a vector, or scalar. If the
6325 : shift/rotate amount is a vector, use the vector/vector shift optabs. */
6326 :
6327 65101 : if ((dt[1] == vect_internal_def
6328 65101 : || dt[1] == vect_induction_def
6329 48449 : || dt[1] == vect_nested_cycle)
6330 16670 : && SLP_TREE_LANES (slp_node) == 1)
6331 : scalar_shift_arg = false;
6332 48490 : else if (dt[1] == vect_constant_def
6333 : || dt[1] == vect_external_def
6334 48490 : || dt[1] == vect_internal_def)
6335 : {
6336 : /* For internal SLP defs we have to make sure we see scalar stmts
6337 : for all vector elements.
6338 : ??? For different vectors we could resort to a different
6339 : scalar shift operand but code-generation below simply always
6340 : takes the first. */
6341 48484 : if (dt[1] == vect_internal_def
6342 48484 : && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
6343 : SLP_TREE_LANES (slp_node)))
6344 : scalar_shift_arg = false;
6345 : /* If the shift amount is computed by a pattern stmt we cannot
6346 : use the scalar amount directly thus give up and use a vector
6347 : shift. */
6348 48474 : else if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
6349 : scalar_shift_arg = false;
6350 : else
6351 : {
6352 48467 : tree l0 = vect_get_slp_scalar_def (slp_op1, 0);
6353 48467 : if (!l0)
6354 : scalar_shift_arg = false;
6355 : else
6356 76283 : for (k = 1; k < SLP_TREE_LANES (slp_node); ++k)
6357 27846 : if (!operand_equal_p (vect_get_slp_scalar_def (slp_op1, k), l0))
6358 : {
6359 : scalar_shift_arg = false;
6360 : break;
6361 : }
6362 : }
6363 : }
6364 : else
6365 : {
6366 6 : if (dump_enabled_p ())
6367 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6368 : "operand mode requires invariant argument.\n");
6369 : return false;
6370 : }
6371 :
6372 : /* Vector shifted by vector. */
6373 65125 : bool was_scalar_shift_arg = scalar_shift_arg;
6374 48467 : if (!scalar_shift_arg)
6375 : {
6376 16658 : optab = optab_for_tree_code (code, vectype, optab_vector);
6377 16658 : if (dump_enabled_p ())
6378 1251 : dump_printf_loc (MSG_NOTE, vect_location,
6379 : "vector/vector shift/rotate found.\n");
6380 :
6381 16658 : if (!op1_vectype)
6382 15 : op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
6383 : slp_op1);
6384 16658 : incompatible_op1_vectype_p
6385 33316 : = (op1_vectype == NULL_TREE
6386 16658 : || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
6387 16658 : TYPE_VECTOR_SUBPARTS (vectype))
6388 33314 : || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
6389 16658 : if (incompatible_op1_vectype_p
6390 7 : && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
6391 1 : || slp_op1->refcnt != 1))
6392 : {
6393 6 : if (dump_enabled_p ())
6394 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6395 : "unusable type for last operand in"
6396 : " vector/vector shift/rotate.\n");
6397 : return false;
6398 : }
6399 : }
6400 : /* See if the machine has a vector shifted by scalar insn and if not
6401 : then see if it has a vector shifted by vector insn. */
6402 : else
6403 : {
6404 48437 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6405 48437 : if (optab
6406 48437 : && can_implement_p (optab, TYPE_MODE (vectype)))
6407 : {
6408 48167 : if (dump_enabled_p ())
6409 5683 : dump_printf_loc (MSG_NOTE, vect_location,
6410 : "vector/scalar shift/rotate found.\n");
6411 : }
6412 : else
6413 : {
6414 270 : optab = optab_for_tree_code (code, vectype, optab_vector);
6415 270 : if (optab
6416 270 : && can_implement_p (optab, TYPE_MODE (vectype)))
6417 : {
6418 0 : scalar_shift_arg = false;
6419 :
6420 0 : if (dump_enabled_p ())
6421 0 : dump_printf_loc (MSG_NOTE, vect_location,
6422 : "vector/vector shift/rotate found.\n");
6423 :
6424 0 : if (!op1_vectype)
6425 0 : op1_vectype = get_vectype_for_scalar_type (vinfo,
6426 0 : TREE_TYPE (op1),
6427 : slp_op1);
6428 :
6429 : /* Unlike the other binary operators, shifts/rotates have
6430 : the rhs being int, instead of the same type as the lhs,
6431 : so make sure the scalar is the right type if we are
6432 : dealing with vectors of long long/long/short/char. */
6433 0 : incompatible_op1_vectype_p
6434 0 : = (!op1_vectype
6435 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype),
6436 0 : TREE_TYPE (op1)));
6437 : if (incompatible_op1_vectype_p
6438 0 : && dt[1] == vect_internal_def)
6439 : {
6440 0 : if (dump_enabled_p ())
6441 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6442 : "unusable type for last operand in"
6443 : " vector/vector shift/rotate.\n");
6444 : return false;
6445 : }
6446 : }
6447 : }
6448 : }
6449 :
6450 : /* Supportable by target? */
6451 65089 : if (!optab)
6452 : {
6453 0 : if (dump_enabled_p ())
6454 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6455 : "no shift optab for %s and %T.\n",
6456 : get_tree_code_name (code), vectype);
6457 : return false;
6458 : }
6459 65089 : vec_mode = TYPE_MODE (vectype);
6460 65089 : icode = (int) optab_handler (optab, vec_mode);
6461 65089 : if (icode == CODE_FOR_nothing)
6462 : {
6463 6560 : if (dump_enabled_p ())
6464 972 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6465 : "shift op not supported by target.\n");
6466 : return false;
6467 : }
6468 : /* vector lowering cannot optimize vector shifts using word arithmetic. */
6469 58529 : if (vect_emulated_vector_p (vectype))
6470 : return false;
6471 :
6472 58529 : if (cost_vec) /* transformation not required. */
6473 : {
6474 49297 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6475 49297 : || ((!scalar_shift_arg || dt[1] == vect_internal_def)
6476 8201 : && (!incompatible_op1_vectype_p
6477 1 : || dt[1] == vect_constant_def)
6478 8201 : && !vect_maybe_update_slp_op_vectype
6479 8201 : (slp_op1,
6480 : incompatible_op1_vectype_p ? vectype : op1_vectype)))
6481 : {
6482 0 : if (dump_enabled_p ())
6483 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6484 : "incompatible vector types for invariants\n");
6485 : return false;
6486 : }
6487 : /* Now adjust the constant shift amount in place. */
6488 49297 : if (incompatible_op1_vectype_p
6489 1 : && dt[1] == vect_constant_def)
6490 : {
6491 1 : unsigned group_size = SLP_TREE_LANES (slp_op1);
6492 2 : gcc_assert (SLP_TREE_SCALAR_OPS (slp_op1).length () == group_size);
6493 5 : for (unsigned i = 0; i < group_size; ++i)
6494 : {
6495 4 : SLP_TREE_SCALAR_OPS (slp_op1)[i]
6496 4 : = fold_convert (TREE_TYPE (vectype),
6497 : SLP_TREE_SCALAR_OPS (slp_op1)[i]);
6498 4 : gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
6499 : == INTEGER_CST));
6500 : }
6501 : }
6502 49297 : SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
6503 49297 : DUMP_VECT_SCOPE ("vectorizable_shift");
6504 49297 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6505 49297 : return true;
6506 : }
6507 :
6508 : /* Transform. */
6509 :
6510 9232 : if (dump_enabled_p ())
6511 2278 : dump_printf_loc (MSG_NOTE, vect_location,
6512 : "transform binary/unary operation.\n");
6513 :
6514 : /* Handle def. */
6515 9232 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6516 :
6517 9232 : unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
6518 9232 : if (scalar_shift_arg && dt[1] != vect_internal_def)
6519 : {
6520 : /* Vector shl and shr insn patterns can be defined with scalar
6521 : operand 2 (shift operand). In this case, use constant or loop
6522 : invariant op1 directly, without extending it to vector mode
6523 : first. */
6524 7047 : optab_op2_mode = insn_data[icode].operand[2].mode;
6525 7047 : if (!VECTOR_MODE_P (optab_op2_mode))
6526 : {
6527 7047 : if (dump_enabled_p ())
6528 2161 : dump_printf_loc (MSG_NOTE, vect_location,
6529 : "operand 1 using scalar mode.\n");
6530 7047 : vec_oprnd1 = op1;
6531 7047 : vec_oprnds1.create (nvectors);
6532 7047 : vec_oprnds1.quick_push (vec_oprnd1);
6533 : /* Store vec_oprnd1 for every vector stmt to be created.
6534 : We check during the analysis that all the shift arguments
6535 : are the same.
6536 : TODO: Allow different constants for different vector
6537 : stmts generated for an SLP instance. */
6538 16118 : for (k = 0; k < nvectors - 1; k++)
6539 2024 : vec_oprnds1.quick_push (vec_oprnd1);
6540 : }
6541 : }
6542 2185 : else if (!scalar_shift_arg && incompatible_op1_vectype_p)
6543 : {
6544 0 : if (was_scalar_shift_arg)
6545 : {
6546 : /* If the argument was the same in all lanes create the
6547 : correctly typed vector shift amount directly. Note
6548 : we made SLP scheduling think we use the original scalars,
6549 : so place the compensation code next to the shift which
6550 : is conservative. See PR119640 where it otherwise breaks. */
6551 0 : op1 = fold_convert (TREE_TYPE (vectype), op1);
6552 0 : op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
6553 : gsi);
6554 0 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
6555 : gsi);
6556 0 : vec_oprnds1.create (nvectors);
6557 0 : for (k = 0; k < nvectors; k++)
6558 0 : vec_oprnds1.quick_push (vec_oprnd1);
6559 : }
6560 0 : else if (dt[1] == vect_constant_def)
6561 : /* The constant shift amount has been adjusted in place. */
6562 : ;
6563 : else
6564 0 : gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
6565 : }
6566 :
6567 : /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6568 : (a special case for certain kind of vector shifts); otherwise,
6569 : operand 1 should be of a vector type (the usual case). */
6570 9232 : vect_get_vec_defs (vinfo, slp_node,
6571 : true, &vec_oprnds0, !vec_oprnd1, &vec_oprnds1);
6572 :
6573 : /* Arguments are ready. Create the new vector stmt. */
6574 33254 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6575 : {
6576 : /* For internal defs where we need to use a scalar shift arg
6577 : extract the first lane. */
6578 14790 : if (scalar_shift_arg && dt[1] == vect_internal_def)
6579 : {
6580 12 : vop1 = vec_oprnds1[0];
6581 12 : new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
6582 12 : gassign *new_stmt
6583 12 : = gimple_build_assign (new_temp,
6584 12 : build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
6585 : vop1,
6586 12 : TYPE_SIZE (TREE_TYPE (new_temp)),
6587 : bitsize_zero_node));
6588 12 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6589 12 : vop1 = new_temp;
6590 12 : }
6591 : else
6592 14778 : vop1 = vec_oprnds1[i];
6593 14790 : gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
6594 14790 : new_temp = make_ssa_name (vec_dest, new_stmt);
6595 14790 : gimple_assign_set_lhs (new_stmt, new_temp);
6596 14790 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6597 14790 : slp_node->push_vec_def (new_stmt);
6598 : }
6599 :
6600 9232 : vec_oprnds0.release ();
6601 9232 : vec_oprnds1.release ();
6602 :
6603 9232 : return true;
6604 : }
6605 :
6606 : /* Function vectorizable_operation.
6607 :
6608 : Check if STMT_INFO performs a binary, unary or ternary operation that can
6609 : be vectorized.
6610 : If COST_VEC is passed, calculate costs but don't change anything,
6611 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6612 : it, and insert it at GSI.
6613 : Return true if STMT_INFO is vectorizable in this way. */
6614 :
6615 : static bool
6616 2697036 : vectorizable_operation (vec_info *vinfo,
6617 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6618 : slp_tree slp_node,
6619 : stmt_vector_for_cost *cost_vec)
6620 : {
6621 2697036 : tree vec_dest;
6622 2697036 : tree scalar_dest;
6623 2697036 : tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
6624 2697036 : tree vectype;
6625 2697036 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6626 2697036 : enum tree_code code, orig_code;
6627 2697036 : machine_mode vec_mode;
6628 2697036 : tree new_temp;
6629 2697036 : int op_type;
6630 2697036 : optab optab;
6631 2697036 : bool target_support_p;
6632 2697036 : enum vect_def_type dt[3]
6633 : = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
6634 2697036 : poly_uint64 nunits_in;
6635 2697036 : poly_uint64 nunits_out;
6636 2697036 : tree vectype_out;
6637 2697036 : int i;
6638 2697036 : vec<tree> vec_oprnds0 = vNULL;
6639 2697036 : vec<tree> vec_oprnds1 = vNULL;
6640 2697036 : vec<tree> vec_oprnds2 = vNULL;
6641 2697036 : tree vop0, vop1, vop2;
6642 2697036 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6643 :
6644 2697036 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6645 : return false;
6646 :
6647 2697036 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6648 251571 : && cost_vec)
6649 : return false;
6650 :
6651 : /* Is STMT a vectorizable binary/unary operation? */
6652 2445465 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6653 2368689 : if (!stmt)
6654 : return false;
6655 :
6656 : /* Loads and stores are handled in vectorizable_{load,store}. */
6657 2368689 : if (STMT_VINFO_DATA_REF (stmt_info))
6658 : return false;
6659 :
6660 950066 : orig_code = code = gimple_assign_rhs_code (stmt);
6661 :
6662 : /* Shifts are handled in vectorizable_shift. */
6663 950066 : if (code == LSHIFT_EXPR
6664 : || code == RSHIFT_EXPR
6665 : || code == LROTATE_EXPR
6666 950066 : || code == RROTATE_EXPR)
6667 : return false;
6668 :
6669 : /* Comparisons are handled in vectorizable_comparison. */
6670 894197 : if (TREE_CODE_CLASS (code) == tcc_comparison)
6671 : return false;
6672 :
6673 : /* Conditions are handled in vectorizable_condition. */
6674 693983 : if (code == COND_EXPR)
6675 : return false;
6676 :
6677 : /* For pointer addition and subtraction, we should use the normal
6678 : plus and minus for the vector operation. */
6679 666757 : if (code == POINTER_PLUS_EXPR)
6680 : code = PLUS_EXPR;
6681 644409 : if (code == POINTER_DIFF_EXPR)
6682 3382 : code = MINUS_EXPR;
6683 :
6684 : /* Support only unary or binary operations. */
6685 666757 : op_type = TREE_CODE_LENGTH (code);
6686 666757 : if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
6687 : {
6688 15 : if (dump_enabled_p ())
6689 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6690 : "num. args = %d (not unary/binary/ternary op).\n",
6691 : op_type);
6692 : return false;
6693 : }
6694 :
6695 666742 : scalar_dest = gimple_assign_lhs (stmt);
6696 666742 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6697 :
6698 : /* Most operations cannot handle bit-precision types without extra
6699 : truncations. */
6700 666742 : bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
6701 655153 : if (!mask_op_p
6702 655153 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
6703 : /* Exception are bitwise binary operations. */
6704 : && code != BIT_IOR_EXPR
6705 2048 : && code != BIT_XOR_EXPR
6706 1179 : && code != BIT_AND_EXPR)
6707 : {
6708 847 : if (dump_enabled_p ())
6709 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6710 : "bit-precision arithmetic not supported.\n");
6711 : return false;
6712 : }
6713 :
6714 665895 : slp_tree slp_op0;
6715 665895 : if (!vect_is_simple_use (vinfo, slp_node,
6716 : 0, &op0, &slp_op0, &dt[0], &vectype))
6717 : {
6718 0 : if (dump_enabled_p ())
6719 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6720 : "use not simple.\n");
6721 : return false;
6722 : }
6723 665895 : bool is_invariant = (dt[0] == vect_external_def
6724 665895 : || dt[0] == vect_constant_def);
6725 : /* If op0 is an external or constant def, infer the vector type
6726 : from the scalar type. */
6727 665895 : if (!vectype)
6728 : {
6729 : /* For boolean type we cannot determine vectype by
6730 : invariant value (don't know whether it is a vector
6731 : of booleans or vector of integers). We use output
6732 : vectype because operations on boolean don't change
6733 : type. */
6734 79058 : if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
6735 : {
6736 2168 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
6737 : {
6738 337 : if (dump_enabled_p ())
6739 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6740 : "not supported operation on bool value.\n");
6741 : return false;
6742 : }
6743 1831 : vectype = vectype_out;
6744 : }
6745 : else
6746 76890 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
6747 : slp_node);
6748 : }
6749 665558 : if (!cost_vec)
6750 119230 : gcc_assert (vectype);
6751 665558 : if (!vectype)
6752 : {
6753 312 : if (dump_enabled_p ())
6754 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6755 : "no vectype for scalar type %T\n",
6756 2 : TREE_TYPE (op0));
6757 :
6758 : return false;
6759 : }
6760 :
6761 665246 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6762 665246 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6763 665246 : if (maybe_ne (nunits_out, nunits_in)
6764 665246 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
6765 : return false;
6766 :
6767 644087 : tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
6768 644087 : slp_tree slp_op1 = NULL, slp_op2 = NULL;
6769 644087 : if (op_type == binary_op || op_type == ternary_op)
6770 : {
6771 561391 : if (!vect_is_simple_use (vinfo, slp_node,
6772 : 1, &op1, &slp_op1, &dt[1], &vectype2))
6773 : {
6774 0 : if (dump_enabled_p ())
6775 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6776 : "use not simple.\n");
6777 : return false;
6778 : }
6779 561391 : is_invariant &= (dt[1] == vect_external_def
6780 561391 : || dt[1] == vect_constant_def);
6781 561391 : if (vectype2
6782 903169 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
6783 341778 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6784 341778 : TREE_TYPE (vectype2))))
6785 : return false;
6786 : }
6787 644087 : if (op_type == ternary_op)
6788 : {
6789 0 : if (!vect_is_simple_use (vinfo, slp_node,
6790 : 2, &op2, &slp_op2, &dt[2], &vectype3))
6791 : {
6792 0 : if (dump_enabled_p ())
6793 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6794 : "use not simple.\n");
6795 : return false;
6796 : }
6797 0 : is_invariant &= (dt[2] == vect_external_def
6798 0 : || dt[2] == vect_constant_def);
6799 0 : if (vectype3
6800 0 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
6801 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6802 0 : TREE_TYPE (vectype3))))
6803 : return false;
6804 : }
6805 :
6806 : /* Multiple types in SLP are handled by creating the appropriate number of
6807 : vectorized stmts for each SLP node. */
6808 644087 : auto vec_num = vect_get_num_copies (vinfo, slp_node);
6809 :
6810 : /* Reject attempts to combine mask types with nonmask types, e.g. if
6811 : we have an AND between a (nonmask) boolean loaded from memory and
6812 : a (mask) boolean result of a comparison.
6813 :
6814 : TODO: We could easily fix these cases up using pattern statements. */
6815 644087 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
6816 977642 : || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
6817 1288174 : || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
6818 : {
6819 0 : if (dump_enabled_p ())
6820 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6821 : "mixed mask and nonmask vector types\n");
6822 : return false;
6823 : }
6824 :
6825 : /* Supportable by target? */
6826 :
6827 644087 : vec_mode = TYPE_MODE (vectype);
6828 644087 : optab = optab_for_tree_code (code, vectype, optab_default);
6829 644087 : if (!optab)
6830 : {
6831 71207 : if (dump_enabled_p ())
6832 6157 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6833 : "no optab for %s and %T.\n",
6834 : get_tree_code_name (code), vectype);
6835 : return false;
6836 : }
6837 572880 : target_support_p = can_implement_p (optab, vec_mode);
6838 :
6839 572880 : bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
6840 572880 : if (!target_support_p || using_emulated_vectors_p)
6841 : {
6842 30682 : if (dump_enabled_p ())
6843 1302 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6844 : "op not supported by target.\n");
6845 : /* When vec_mode is not a vector mode and we verified ops we
6846 : do not have to lower like AND are natively supported let
6847 : those through even when the mode isn't word_mode. For
6848 : ops we have to lower the lowering code assumes we are
6849 : dealing with word_mode. */
6850 61364 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
6851 30532 : || !GET_MODE_SIZE (vec_mode).is_constant ()
6852 30532 : || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6853 25327 : || !target_support_p)
6854 65591 : && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
6855 : /* Check only during analysis. */
6856 43021 : || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
6857 : {
6858 30080 : if (dump_enabled_p ())
6859 1300 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
6860 : return false;
6861 : }
6862 602 : if (dump_enabled_p ())
6863 2 : dump_printf_loc (MSG_NOTE, vect_location,
6864 : "proceeding using word mode.\n");
6865 : using_emulated_vectors_p = true;
6866 : }
6867 :
6868 542800 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
6869 542800 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
6870 438789 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
6871 542800 : internal_fn cond_fn = get_conditional_internal_fn (code);
6872 542800 : internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
6873 :
6874 : /* If operating on inactive elements could generate spurious traps,
6875 : we need to restrict the operation to active lanes. Note that this
6876 : specifically doesn't apply to unhoisted invariants, since they
6877 : operate on the same value for every lane.
6878 :
6879 : Similarly, if this operation is part of a reduction, a fully-masked
6880 : loop should only change the active lanes of the reduction chain,
6881 : keeping the inactive lanes as-is. */
6882 513947 : bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
6883 987428 : || reduc_idx >= 0);
6884 :
6885 542800 : if (cost_vec) /* transformation not required. */
6886 : {
6887 423570 : if (loop_vinfo
6888 332367 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
6889 92891 : && mask_out_inactive)
6890 : {
6891 19983 : if (cond_len_fn != IFN_LAST
6892 19983 : && direct_internal_fn_supported_p (cond_len_fn, vectype,
6893 : OPTIMIZE_FOR_SPEED))
6894 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
6895 : 1);
6896 19983 : else if (cond_fn != IFN_LAST
6897 19983 : && direct_internal_fn_supported_p (cond_fn, vectype,
6898 : OPTIMIZE_FOR_SPEED))
6899 8182 : vect_record_loop_mask (loop_vinfo, masks, vec_num,
6900 : vectype, NULL);
6901 : else
6902 : {
6903 11801 : if (dump_enabled_p ())
6904 613 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6905 : "can't use a fully-masked loop because no"
6906 : " conditional operation is available.\n");
6907 11801 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6908 : }
6909 : }
6910 :
6911 : /* Put types on constant and invariant SLP children. */
6912 423570 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6913 423502 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
6914 846978 : || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
6915 : {
6916 162 : if (dump_enabled_p ())
6917 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6918 : "incompatible vector types for invariants\n");
6919 : return false;
6920 : }
6921 :
6922 423408 : SLP_TREE_TYPE (slp_node) = op_vec_info_type;
6923 423408 : DUMP_VECT_SCOPE ("vectorizable_operation");
6924 423408 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6925 423408 : if (using_emulated_vectors_p)
6926 : {
6927 : /* The above vect_model_simple_cost call handles constants
6928 : in the prologue and (mis-)costs one of the stmts as
6929 : vector stmt. See below for the actual lowering that will
6930 : be applied. */
6931 600 : unsigned n = vect_get_num_copies (vinfo, slp_node);
6932 600 : switch (code)
6933 : {
6934 217 : case PLUS_EXPR:
6935 217 : n *= 5;
6936 217 : break;
6937 346 : case MINUS_EXPR:
6938 346 : n *= 6;
6939 346 : break;
6940 0 : case NEGATE_EXPR:
6941 0 : n *= 4;
6942 0 : break;
6943 : default:
6944 : /* Bit operations do not have extra cost and are accounted
6945 : as vector stmt by vect_model_simple_cost. */
6946 : n = 0;
6947 : break;
6948 : }
6949 563 : if (n != 0)
6950 : {
6951 : /* We also need to materialize two large constants. */
6952 563 : record_stmt_cost (cost_vec, 2, scalar_stmt, slp_node,
6953 : 0, vect_prologue);
6954 563 : record_stmt_cost (cost_vec, n, scalar_stmt, slp_node,
6955 : 0, vect_body);
6956 : }
6957 : }
6958 423408 : return true;
6959 : }
6960 :
6961 : /* Transform. */
6962 :
6963 119230 : if (dump_enabled_p ())
6964 17516 : dump_printf_loc (MSG_NOTE, vect_location,
6965 : "transform binary/unary operation.\n");
6966 :
6967 119230 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
6968 106422 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
6969 :
6970 : /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6971 : vectors with unsigned elements, but the result is signed. So, we
6972 : need to compute the MINUS_EXPR into vectype temporary and
6973 : VIEW_CONVERT_EXPR it into the final vectype_out result. */
6974 119230 : tree vec_cvt_dest = NULL_TREE;
6975 119230 : if (orig_code == POINTER_DIFF_EXPR)
6976 : {
6977 194 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6978 194 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6979 : }
6980 : /* For reduction operations with undefined overflow behavior make sure to
6981 : pun them to unsigned since we change the order of evaluation.
6982 : ??? Avoid for in-order reductions? */
6983 119036 : else if (arith_code_with_undefined_signed_overflow (orig_code)
6984 100162 : && ANY_INTEGRAL_TYPE_P (vectype)
6985 48804 : && TYPE_OVERFLOW_UNDEFINED (vectype)
6986 145129 : && SLP_TREE_REDUC_IDX (slp_node) != -1)
6987 : {
6988 2489 : gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
6989 : || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
6990 2489 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6991 2489 : vectype = unsigned_type_for (vectype);
6992 2489 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6993 : }
6994 : /* Handle def. */
6995 : else
6996 116547 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
6997 :
6998 119230 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
6999 : slp_op1, &vec_oprnds1, slp_op2, &vec_oprnds2);
7000 : /* Arguments are ready. Create the new vector stmt. */
7001 381987 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
7002 : {
7003 143527 : gimple *new_stmt = NULL;
7004 287054 : vop1 = ((op_type == binary_op || op_type == ternary_op)
7005 143527 : ? vec_oprnds1[i] : NULL_TREE);
7006 143527 : vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
7007 :
7008 143527 : if (vec_cvt_dest
7009 143527 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
7010 : {
7011 2946 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
7012 2946 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
7013 : new_temp);
7014 2946 : new_temp = make_ssa_name (vec_dest, new_stmt);
7015 2946 : gimple_assign_set_lhs (new_stmt, new_temp);
7016 2946 : vect_finish_stmt_generation (vinfo, stmt_info,
7017 : new_stmt, gsi);
7018 2946 : vop0 = new_temp;
7019 : }
7020 143527 : if (vop1
7021 140845 : && vec_cvt_dest
7022 146682 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
7023 : {
7024 2946 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
7025 2946 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
7026 : new_temp);
7027 2946 : new_temp = make_ssa_name (vec_dest, new_stmt);
7028 2946 : gimple_assign_set_lhs (new_stmt, new_temp);
7029 2946 : vect_finish_stmt_generation (vinfo, stmt_info,
7030 : new_stmt, gsi);
7031 2946 : vop1 = new_temp;
7032 : }
7033 143527 : if (vop2
7034 0 : && vec_cvt_dest
7035 143527 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
7036 : {
7037 0 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
7038 0 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
7039 : new_temp);
7040 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
7041 0 : gimple_assign_set_lhs (new_stmt, new_temp);
7042 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7043 : new_stmt, gsi);
7044 0 : vop2 = new_temp;
7045 : }
7046 :
7047 143527 : if (using_emulated_vectors_p)
7048 : {
7049 : /* Lower the operation. This follows vector lowering. */
7050 2 : tree word_type = build_nonstandard_integer_type
7051 2 : (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
7052 2 : tree wvop0 = make_ssa_name (word_type);
7053 2 : new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
7054 : build1 (VIEW_CONVERT_EXPR,
7055 : word_type, vop0));
7056 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7057 2 : tree wvop1 = NULL_TREE;
7058 2 : if (vop1)
7059 : {
7060 2 : wvop1 = make_ssa_name (word_type);
7061 2 : new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
7062 : build1 (VIEW_CONVERT_EXPR,
7063 : word_type, vop1));
7064 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7065 : }
7066 :
7067 2 : tree result_low;
7068 2 : if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
7069 : {
7070 1 : unsigned int width = vector_element_bits (vectype);
7071 1 : tree inner_type = TREE_TYPE (vectype);
7072 1 : HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
7073 1 : tree low_bits
7074 1 : = build_replicated_int_cst (word_type, width, max >> 1);
7075 1 : tree high_bits
7076 2 : = build_replicated_int_cst (word_type,
7077 1 : width, max & ~(max >> 1));
7078 1 : tree signs;
7079 1 : if (code == PLUS_EXPR || code == MINUS_EXPR)
7080 : {
7081 1 : signs = make_ssa_name (word_type);
7082 1 : new_stmt = gimple_build_assign (signs,
7083 : BIT_XOR_EXPR, wvop0, wvop1);
7084 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7085 1 : tree b_low = make_ssa_name (word_type);
7086 1 : new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
7087 : wvop1, low_bits);
7088 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7089 1 : tree a_low = make_ssa_name (word_type);
7090 1 : if (code == PLUS_EXPR)
7091 1 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7092 : wvop0, low_bits);
7093 : else
7094 0 : new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
7095 : wvop0, high_bits);
7096 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7097 1 : if (code == MINUS_EXPR)
7098 : {
7099 0 : new_stmt = gimple_build_assign (NULL_TREE,
7100 : BIT_NOT_EXPR, signs);
7101 0 : signs = make_ssa_name (word_type);
7102 0 : gimple_assign_set_lhs (new_stmt, signs);
7103 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7104 : new_stmt, gsi);
7105 : }
7106 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7107 : signs, high_bits);
7108 1 : signs = make_ssa_name (word_type);
7109 1 : gimple_assign_set_lhs (new_stmt, signs);
7110 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7111 1 : result_low = make_ssa_name (word_type);
7112 1 : new_stmt = gimple_build_assign (result_low, code,
7113 : a_low, b_low);
7114 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7115 : }
7116 : else /* if (code == NEGATE_EXPR) */
7117 : {
7118 0 : tree a_low = make_ssa_name (word_type);
7119 0 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7120 : wvop0, low_bits);
7121 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7122 0 : signs = make_ssa_name (word_type);
7123 0 : new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
7124 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7125 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7126 : signs, high_bits);
7127 0 : signs = make_ssa_name (word_type);
7128 0 : gimple_assign_set_lhs (new_stmt, signs);
7129 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7130 0 : result_low = make_ssa_name (word_type);
7131 0 : new_stmt = gimple_build_assign (result_low,
7132 : MINUS_EXPR, high_bits, a_low);
7133 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7134 : }
7135 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
7136 : result_low, signs);
7137 1 : result_low = make_ssa_name (word_type);
7138 1 : gimple_assign_set_lhs (new_stmt, result_low);
7139 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7140 : }
7141 : else
7142 : {
7143 1 : new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
7144 1 : result_low = make_ssa_name (word_type);
7145 1 : gimple_assign_set_lhs (new_stmt, result_low);
7146 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7147 :
7148 : }
7149 2 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
7150 : build1 (VIEW_CONVERT_EXPR,
7151 : vectype, result_low));
7152 2 : new_temp = make_ssa_name (vectype);
7153 2 : gimple_assign_set_lhs (new_stmt, new_temp);
7154 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7155 : }
7156 143525 : else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
7157 : {
7158 16 : tree mask;
7159 16 : if (masked_loop_p)
7160 16 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7161 : vec_num, vectype, i);
7162 : else
7163 : /* Dummy mask. */
7164 0 : mask = build_minus_one_cst (truth_type_for (vectype));
7165 16 : auto_vec<tree> vops (6);
7166 16 : vops.quick_push (mask);
7167 16 : vops.quick_push (vop0);
7168 16 : if (vop1)
7169 16 : vops.quick_push (vop1);
7170 16 : if (vop2)
7171 0 : vops.quick_push (vop2);
7172 16 : if (reduc_idx >= 0)
7173 : {
7174 : /* Perform the operation on active elements only and take
7175 : inactive elements from the reduction chain input. */
7176 8 : gcc_assert (!vop2);
7177 8 : vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
7178 : }
7179 : else
7180 : {
7181 8 : auto else_value = targetm.preferred_else_value
7182 8 : (cond_fn, vectype, vops.length () - 1, &vops[1]);
7183 8 : vops.quick_push (else_value);
7184 : }
7185 16 : if (len_loop_p)
7186 : {
7187 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
7188 0 : vec_num, vectype, i, 1, true);
7189 0 : signed char biasval
7190 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
7191 0 : tree bias = build_int_cst (intQI_type_node, biasval);
7192 0 : vops.quick_push (len);
7193 0 : vops.quick_push (bias);
7194 : }
7195 16 : gcall *call
7196 16 : = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
7197 : : cond_len_fn,
7198 : vops);
7199 16 : new_temp = make_ssa_name (vec_dest, call);
7200 16 : gimple_call_set_lhs (call, new_temp);
7201 16 : gimple_call_set_nothrow (call, true);
7202 16 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
7203 16 : new_stmt = call;
7204 16 : }
7205 : else
7206 : {
7207 143509 : tree mask = NULL_TREE;
7208 : /* When combining two masks check if either of them is elsewhere
7209 : combined with a loop mask, if that's the case we can mark that the
7210 : new combined mask doesn't need to be combined with a loop mask. */
7211 143509 : if (masked_loop_p
7212 143509 : && code == BIT_AND_EXPR
7213 143509 : && VECTOR_BOOLEAN_TYPE_P (vectype))
7214 : {
7215 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
7216 : {
7217 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7218 : vec_num, vectype, i);
7219 :
7220 0 : vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7221 : vop0, gsi);
7222 : }
7223 :
7224 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
7225 : {
7226 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7227 : vec_num, vectype, i);
7228 :
7229 0 : vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7230 : vop1, gsi);
7231 : }
7232 : }
7233 :
7234 143509 : new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
7235 143509 : new_temp = make_ssa_name (vec_dest, new_stmt);
7236 143509 : gimple_assign_set_lhs (new_stmt, new_temp);
7237 143509 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7238 143509 : if (using_emulated_vectors_p)
7239 : suppress_warning (new_stmt, OPT_Wvector_operation_performance);
7240 :
7241 : /* Enter the combined value into the vector cond hash so we don't
7242 : AND it with a loop mask again. */
7243 143509 : if (mask)
7244 0 : loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
7245 : }
7246 :
7247 143527 : if (vec_cvt_dest)
7248 : {
7249 3155 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
7250 3155 : new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
7251 : new_temp);
7252 3155 : new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
7253 3155 : gimple_assign_set_lhs (new_stmt, new_temp);
7254 3155 : vect_finish_stmt_generation (vinfo, stmt_info,
7255 : new_stmt, gsi);
7256 : }
7257 :
7258 143527 : slp_node->push_vec_def (new_stmt);
7259 : }
7260 :
7261 119230 : vec_oprnds0.release ();
7262 119230 : vec_oprnds1.release ();
7263 119230 : vec_oprnds2.release ();
7264 :
7265 119230 : return true;
7266 : }
7267 :
7268 : /* A helper function to ensure data reference DR_INFO's base alignment. */
7269 :
7270 : static void
7271 708178 : ensure_base_align (dr_vec_info *dr_info)
7272 : {
7273 : /* Alignment is only analyzed for the first element of a DR group,
7274 : use that to look at base alignment we need to enforce. */
7275 708178 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
7276 583582 : dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
7277 :
7278 708178 : gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
7279 :
7280 708178 : if (dr_info->base_misaligned)
7281 : {
7282 113568 : tree base_decl = dr_info->base_decl;
7283 :
7284 : // We should only be able to increase the alignment of a base object if
7285 : // we know what its new alignment should be at compile time.
7286 113568 : unsigned HOST_WIDE_INT align_base_to =
7287 113568 : DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
7288 :
7289 113568 : if (decl_in_symtab_p (base_decl))
7290 3261 : symtab_node::get (base_decl)->increase_alignment (align_base_to);
7291 110307 : else if (DECL_ALIGN (base_decl) < align_base_to)
7292 : {
7293 91975 : SET_DECL_ALIGN (base_decl, align_base_to);
7294 91975 : DECL_USER_ALIGN (base_decl) = 1;
7295 : }
7296 113568 : dr_info->base_misaligned = false;
7297 : }
7298 708178 : }
7299 :
7300 :
7301 : /* Function get_group_alias_ptr_type.
7302 :
7303 : Return the alias type for the group starting at FIRST_STMT_INFO. */
7304 :
7305 : static tree
7306 1659002 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
7307 : {
7308 1659002 : struct data_reference *first_dr, *next_dr;
7309 :
7310 1659002 : first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
7311 1659002 : stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
7312 3927520 : while (next_stmt_info)
7313 : {
7314 2456965 : next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
7315 4913930 : if (get_alias_set (DR_REF (first_dr))
7316 2456965 : != get_alias_set (DR_REF (next_dr)))
7317 : {
7318 188447 : if (dump_enabled_p ())
7319 30 : dump_printf_loc (MSG_NOTE, vect_location,
7320 : "conflicting alias set types.\n");
7321 188447 : return ptr_type_node;
7322 : }
7323 2268518 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7324 : }
7325 1470555 : return reference_alias_ptr_type (DR_REF (first_dr));
7326 : }
7327 :
7328 :
7329 : /* Function scan_operand_equal_p.
7330 :
7331 : Helper function for check_scan_store. Compare two references
7332 : with .GOMP_SIMD_LANE bases. */
7333 :
7334 : static bool
7335 1284 : scan_operand_equal_p (tree ref1, tree ref2)
7336 : {
7337 1284 : tree ref[2] = { ref1, ref2 };
7338 1284 : poly_int64 bitsize[2], bitpos[2];
7339 : tree offset[2], base[2];
7340 3852 : for (int i = 0; i < 2; ++i)
7341 : {
7342 2568 : machine_mode mode;
7343 2568 : int unsignedp, reversep, volatilep = 0;
7344 2568 : base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
7345 : &offset[i], &mode, &unsignedp,
7346 : &reversep, &volatilep);
7347 2568 : if (reversep || volatilep || maybe_ne (bitpos[i], 0))
7348 0 : return false;
7349 2568 : if (TREE_CODE (base[i]) == MEM_REF
7350 42 : && offset[i] == NULL_TREE
7351 2610 : && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
7352 : {
7353 42 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
7354 42 : if (is_gimple_assign (def_stmt)
7355 42 : && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
7356 42 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
7357 84 : && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
7358 : {
7359 42 : if (maybe_ne (mem_ref_offset (base[i]), 0))
7360 : return false;
7361 42 : base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
7362 42 : offset[i] = gimple_assign_rhs2 (def_stmt);
7363 : }
7364 : }
7365 : }
7366 :
7367 1284 : if (!operand_equal_p (base[0], base[1], 0))
7368 : return false;
7369 934 : if (maybe_ne (bitsize[0], bitsize[1]))
7370 : return false;
7371 934 : if (offset[0] != offset[1])
7372 : {
7373 916 : if (!offset[0] || !offset[1])
7374 : return false;
7375 916 : if (!operand_equal_p (offset[0], offset[1], 0))
7376 : {
7377 : tree step[2];
7378 0 : for (int i = 0; i < 2; ++i)
7379 : {
7380 0 : step[i] = integer_one_node;
7381 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7382 : {
7383 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7384 0 : if (is_gimple_assign (def_stmt)
7385 0 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
7386 0 : && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
7387 : == INTEGER_CST))
7388 : {
7389 0 : step[i] = gimple_assign_rhs2 (def_stmt);
7390 0 : offset[i] = gimple_assign_rhs1 (def_stmt);
7391 : }
7392 : }
7393 0 : else if (TREE_CODE (offset[i]) == MULT_EXPR)
7394 : {
7395 0 : step[i] = TREE_OPERAND (offset[i], 1);
7396 0 : offset[i] = TREE_OPERAND (offset[i], 0);
7397 : }
7398 0 : tree rhs1 = NULL_TREE;
7399 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7400 : {
7401 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7402 0 : if (gimple_assign_cast_p (def_stmt))
7403 0 : rhs1 = gimple_assign_rhs1 (def_stmt);
7404 : }
7405 0 : else if (CONVERT_EXPR_P (offset[i]))
7406 0 : rhs1 = TREE_OPERAND (offset[i], 0);
7407 0 : if (rhs1
7408 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7409 0 : && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
7410 0 : && (TYPE_PRECISION (TREE_TYPE (offset[i]))
7411 0 : >= TYPE_PRECISION (TREE_TYPE (rhs1))))
7412 0 : offset[i] = rhs1;
7413 : }
7414 0 : if (!operand_equal_p (offset[0], offset[1], 0)
7415 0 : || !operand_equal_p (step[0], step[1], 0))
7416 0 : return false;
7417 : }
7418 : }
7419 : return true;
7420 : }
7421 :
7422 :
7423 : enum scan_store_kind {
7424 : /* Normal permutation. */
7425 : scan_store_kind_perm,
7426 :
7427 : /* Whole vector left shift permutation with zero init. */
7428 : scan_store_kind_lshift_zero,
7429 :
7430 : /* Whole vector left shift permutation and VEC_COND_EXPR. */
7431 : scan_store_kind_lshift_cond
7432 : };
7433 :
7434 : /* Function check_scan_store.
7435 :
7436 : Verify if we can perform the needed permutations or whole vector shifts.
7437 : Return -1 on failure, otherwise exact log2 of vectype's nunits.
7438 : USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7439 : to do at each step. */
7440 :
7441 : static int
7442 1024 : scan_store_can_perm_p (tree vectype, tree init,
7443 : vec<enum scan_store_kind> *use_whole_vector = NULL)
7444 : {
7445 1024 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7446 1024 : unsigned HOST_WIDE_INT nunits;
7447 1024 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7448 : return -1;
7449 1024 : int units_log2 = exact_log2 (nunits);
7450 1024 : if (units_log2 <= 0)
7451 : return -1;
7452 :
7453 : int i;
7454 : enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
7455 4784 : for (i = 0; i <= units_log2; ++i)
7456 : {
7457 3760 : unsigned HOST_WIDE_INT j, k;
7458 3760 : enum scan_store_kind kind = scan_store_kind_perm;
7459 3760 : vec_perm_builder sel (nunits, nunits, 1);
7460 3760 : sel.quick_grow (nunits);
7461 3760 : if (i == units_log2)
7462 : {
7463 9728 : for (j = 0; j < nunits; ++j)
7464 8704 : sel[j] = nunits - 1;
7465 : }
7466 : else
7467 : {
7468 10416 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7469 7680 : sel[j] = j;
7470 26416 : for (k = 0; j < nunits; ++j, ++k)
7471 23680 : sel[j] = nunits + k;
7472 : }
7473 6496 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7474 3760 : if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
7475 : {
7476 0 : if (i == units_log2)
7477 : return -1;
7478 :
7479 0 : if (whole_vector_shift_kind == scan_store_kind_perm)
7480 : {
7481 0 : if (!can_implement_p (vec_shl_optab, vec_mode))
7482 : return -1;
7483 0 : whole_vector_shift_kind = scan_store_kind_lshift_zero;
7484 : /* Whole vector shifts shift in zeros, so if init is all zero
7485 : constant, there is no need to do anything further. */
7486 0 : if ((TREE_CODE (init) != INTEGER_CST
7487 0 : && TREE_CODE (init) != REAL_CST)
7488 0 : || !initializer_zerop (init))
7489 : {
7490 0 : tree masktype = truth_type_for (vectype);
7491 0 : if (!expand_vec_cond_expr_p (vectype, masktype))
7492 : return -1;
7493 : whole_vector_shift_kind = scan_store_kind_lshift_cond;
7494 : }
7495 : }
7496 0 : kind = whole_vector_shift_kind;
7497 : }
7498 3760 : if (use_whole_vector)
7499 : {
7500 1880 : if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
7501 0 : use_whole_vector->safe_grow_cleared (i, true);
7502 5640 : if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
7503 0 : use_whole_vector->safe_push (kind);
7504 : }
7505 3760 : }
7506 :
7507 : return units_log2;
7508 : }
7509 :
7510 :
7511 : /* Function check_scan_store.
7512 :
7513 : Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7514 :
7515 : static bool
7516 1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
7517 : enum vect_def_type rhs_dt, slp_tree slp_node,
7518 : slp_tree mask_node,
7519 : vect_memory_access_type memory_access_type)
7520 : {
7521 1076 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7522 1076 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7523 1076 : tree ref_type;
7524 :
7525 1076 : gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
7526 1076 : if (SLP_TREE_LANES (slp_node) > 1
7527 1076 : || mask_node
7528 1076 : || memory_access_type != VMAT_CONTIGUOUS
7529 1076 : || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
7530 1076 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
7531 1076 : || loop_vinfo == NULL
7532 1076 : || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
7533 1076 : || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
7534 1076 : || STMT_VINFO_GROUPED_ACCESS (stmt_info)
7535 1076 : || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
7536 1076 : || !integer_zerop (DR_INIT (dr_info->dr))
7537 1076 : || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
7538 2152 : || !alias_sets_conflict_p (get_alias_set (vectype),
7539 1076 : get_alias_set (TREE_TYPE (ref_type))))
7540 : {
7541 0 : if (dump_enabled_p ())
7542 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7543 : "unsupported OpenMP scan store.\n");
7544 : return false;
7545 : }
7546 :
7547 : /* We need to pattern match code built by OpenMP lowering and simplified
7548 : by following optimizations into something we can handle.
7549 : #pragma omp simd reduction(inscan,+:r)
7550 : for (...)
7551 : {
7552 : r += something ();
7553 : #pragma omp scan inclusive (r)
7554 : use (r);
7555 : }
7556 : shall have body with:
7557 : // Initialization for input phase, store the reduction initializer:
7558 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7559 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7560 : D.2042[_21] = 0;
7561 : // Actual input phase:
7562 : ...
7563 : r.0_5 = D.2042[_20];
7564 : _6 = _4 + r.0_5;
7565 : D.2042[_20] = _6;
7566 : // Initialization for scan phase:
7567 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7568 : _26 = D.2043[_25];
7569 : _27 = D.2042[_25];
7570 : _28 = _26 + _27;
7571 : D.2043[_25] = _28;
7572 : D.2042[_25] = _28;
7573 : // Actual scan phase:
7574 : ...
7575 : r.1_8 = D.2042[_20];
7576 : ...
7577 : The "omp simd array" variable D.2042 holds the privatized copy used
7578 : inside of the loop and D.2043 is another one that holds copies of
7579 : the current original list item. The separate GOMP_SIMD_LANE ifn
7580 : kinds are there in order to allow optimizing the initializer store
7581 : and combiner sequence, e.g. if it is originally some C++ish user
7582 : defined reduction, but allow the vectorizer to pattern recognize it
7583 : and turn into the appropriate vectorized scan.
7584 :
7585 : For exclusive scan, this is slightly different:
7586 : #pragma omp simd reduction(inscan,+:r)
7587 : for (...)
7588 : {
7589 : use (r);
7590 : #pragma omp scan exclusive (r)
7591 : r += something ();
7592 : }
7593 : shall have body with:
7594 : // Initialization for input phase, store the reduction initializer:
7595 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7596 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7597 : D.2042[_21] = 0;
7598 : // Actual input phase:
7599 : ...
7600 : r.0_5 = D.2042[_20];
7601 : _6 = _4 + r.0_5;
7602 : D.2042[_20] = _6;
7603 : // Initialization for scan phase:
7604 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7605 : _26 = D.2043[_25];
7606 : D.2044[_25] = _26;
7607 : _27 = D.2042[_25];
7608 : _28 = _26 + _27;
7609 : D.2043[_25] = _28;
7610 : // Actual scan phase:
7611 : ...
7612 : r.1_8 = D.2044[_20];
7613 : ... */
7614 :
7615 1076 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
7616 : {
7617 : /* Match the D.2042[_21] = 0; store above. Just require that
7618 : it is a constant or external definition store. */
7619 564 : if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
7620 : {
7621 0 : fail_init:
7622 0 : if (dump_enabled_p ())
7623 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7624 : "unsupported OpenMP scan initializer store.\n");
7625 : return false;
7626 : }
7627 :
7628 564 : if (! loop_vinfo->scan_map)
7629 322 : loop_vinfo->scan_map = new hash_map<tree, tree>;
7630 564 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7631 564 : tree &cached = loop_vinfo->scan_map->get_or_insert (var);
7632 564 : if (cached)
7633 0 : goto fail_init;
7634 564 : cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
7635 :
7636 : /* These stores can be vectorized normally. */
7637 564 : return true;
7638 : }
7639 :
7640 512 : if (rhs_dt != vect_internal_def)
7641 : {
7642 0 : fail:
7643 0 : if (dump_enabled_p ())
7644 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7645 : "unsupported OpenMP scan combiner pattern.\n");
7646 : return false;
7647 : }
7648 :
7649 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7650 512 : tree rhs = gimple_assign_rhs1 (stmt);
7651 512 : if (TREE_CODE (rhs) != SSA_NAME)
7652 0 : goto fail;
7653 :
7654 512 : gimple *other_store_stmt = NULL;
7655 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7656 512 : bool inscan_var_store
7657 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7658 :
7659 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7660 : {
7661 252 : if (!inscan_var_store)
7662 : {
7663 126 : use_operand_p use_p;
7664 126 : imm_use_iterator iter;
7665 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7666 : {
7667 252 : gimple *use_stmt = USE_STMT (use_p);
7668 252 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7669 126 : continue;
7670 126 : if (gimple_bb (use_stmt) != gimple_bb (stmt)
7671 126 : || !is_gimple_assign (use_stmt)
7672 126 : || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
7673 126 : || other_store_stmt
7674 252 : || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
7675 0 : goto fail;
7676 126 : other_store_stmt = use_stmt;
7677 0 : }
7678 126 : if (other_store_stmt == NULL)
7679 0 : goto fail;
7680 126 : rhs = gimple_assign_lhs (other_store_stmt);
7681 126 : if (!single_imm_use (rhs, &use_p, &other_store_stmt))
7682 0 : goto fail;
7683 : }
7684 : }
7685 260 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
7686 : {
7687 260 : use_operand_p use_p;
7688 260 : imm_use_iterator iter;
7689 780 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7690 : {
7691 520 : gimple *use_stmt = USE_STMT (use_p);
7692 520 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7693 260 : continue;
7694 260 : if (other_store_stmt)
7695 0 : goto fail;
7696 260 : other_store_stmt = use_stmt;
7697 260 : }
7698 : }
7699 : else
7700 0 : goto fail;
7701 :
7702 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7703 512 : if (gimple_bb (def_stmt) != gimple_bb (stmt)
7704 512 : || !is_gimple_assign (def_stmt)
7705 1024 : || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
7706 0 : goto fail;
7707 :
7708 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7709 : /* For pointer addition, we should use the normal plus for the vector
7710 : operation. */
7711 512 : switch (code)
7712 : {
7713 0 : case POINTER_PLUS_EXPR:
7714 0 : code = PLUS_EXPR;
7715 0 : break;
7716 0 : case MULT_HIGHPART_EXPR:
7717 0 : goto fail;
7718 : default:
7719 : break;
7720 : }
7721 512 : if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
7722 0 : goto fail;
7723 :
7724 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7725 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7726 512 : if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
7727 0 : goto fail;
7728 :
7729 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7730 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7731 512 : if (gimple_bb (load1_stmt) != gimple_bb (stmt)
7732 512 : || !gimple_assign_load_p (load1_stmt)
7733 512 : || gimple_bb (load2_stmt) != gimple_bb (stmt)
7734 1024 : || !gimple_assign_load_p (load2_stmt))
7735 0 : goto fail;
7736 :
7737 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7738 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7739 512 : if (load1_stmt_info == NULL
7740 512 : || load2_stmt_info == NULL
7741 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
7742 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
7743 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
7744 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7745 0 : goto fail;
7746 :
7747 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
7748 : {
7749 126 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7750 126 : if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
7751 126 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
7752 0 : goto fail;
7753 126 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7754 126 : tree lrhs;
7755 126 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7756 : lrhs = rhs1;
7757 : else
7758 16 : lrhs = rhs2;
7759 126 : use_operand_p use_p;
7760 126 : imm_use_iterator iter;
7761 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
7762 : {
7763 252 : gimple *use_stmt = USE_STMT (use_p);
7764 252 : if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
7765 126 : continue;
7766 126 : if (other_store_stmt)
7767 0 : goto fail;
7768 126 : other_store_stmt = use_stmt;
7769 126 : }
7770 : }
7771 :
7772 512 : if (other_store_stmt == NULL)
7773 0 : goto fail;
7774 512 : if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
7775 512 : || !gimple_store_p (other_store_stmt))
7776 0 : goto fail;
7777 :
7778 512 : stmt_vec_info other_store_stmt_info
7779 512 : = loop_vinfo->lookup_stmt (other_store_stmt);
7780 512 : if (other_store_stmt_info == NULL
7781 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
7782 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7783 0 : goto fail;
7784 :
7785 512 : gimple *stmt1 = stmt;
7786 512 : gimple *stmt2 = other_store_stmt;
7787 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7788 : std::swap (stmt1, stmt2);
7789 512 : if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
7790 : gimple_assign_rhs1 (load2_stmt)))
7791 : {
7792 162 : std::swap (rhs1, rhs2);
7793 162 : std::swap (load1_stmt, load2_stmt);
7794 162 : std::swap (load1_stmt_info, load2_stmt_info);
7795 : }
7796 512 : if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
7797 : gimple_assign_rhs1 (load1_stmt)))
7798 0 : goto fail;
7799 :
7800 512 : tree var3 = NULL_TREE;
7801 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
7802 512 : && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
7803 : gimple_assign_rhs1 (load2_stmt)))
7804 0 : goto fail;
7805 512 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7806 : {
7807 252 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7808 252 : if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
7809 252 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
7810 0 : goto fail;
7811 252 : var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7812 252 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
7813 252 : || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
7814 504 : || lookup_attribute ("omp simd inscan exclusive",
7815 252 : DECL_ATTRIBUTES (var3)))
7816 0 : goto fail;
7817 : }
7818 :
7819 512 : dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
7820 512 : if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
7821 512 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
7822 0 : goto fail;
7823 :
7824 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7825 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
7826 512 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
7827 512 : || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
7828 1024 : || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7829 512 : == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
7830 0 : goto fail;
7831 :
7832 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7833 256 : std::swap (var1, var2);
7834 :
7835 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7836 : {
7837 252 : if (!lookup_attribute ("omp simd inscan exclusive",
7838 252 : DECL_ATTRIBUTES (var1)))
7839 0 : goto fail;
7840 252 : var1 = var3;
7841 : }
7842 :
7843 512 : if (loop_vinfo->scan_map == NULL)
7844 0 : goto fail;
7845 512 : tree *init = loop_vinfo->scan_map->get (var1);
7846 512 : if (init == NULL)
7847 0 : goto fail;
7848 :
7849 : /* The IL is as expected, now check if we can actually vectorize it.
7850 : Inclusive scan:
7851 : _26 = D.2043[_25];
7852 : _27 = D.2042[_25];
7853 : _28 = _26 + _27;
7854 : D.2043[_25] = _28;
7855 : D.2042[_25] = _28;
7856 : should be vectorized as (where _40 is the vectorized rhs
7857 : from the D.2042[_21] = 0; store):
7858 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7859 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7860 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7861 : _33 = _31 + _32;
7862 : // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7863 : _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7864 : _35 = _33 + _34;
7865 : // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7866 : // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7867 : _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7868 : _37 = _35 + _36;
7869 : // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7870 : // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7871 : _38 = _30 + _37;
7872 : _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7873 : MEM <vector(8) int> [(int *)&D.2043] = _39;
7874 : MEM <vector(8) int> [(int *)&D.2042] = _38;
7875 : Exclusive scan:
7876 : _26 = D.2043[_25];
7877 : D.2044[_25] = _26;
7878 : _27 = D.2042[_25];
7879 : _28 = _26 + _27;
7880 : D.2043[_25] = _28;
7881 : should be vectorized as (where _40 is the vectorized rhs
7882 : from the D.2042[_21] = 0; store):
7883 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7884 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7885 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7886 : _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7887 : _34 = _32 + _33;
7888 : // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7889 : // _31[3]+_31[4], ... _31[5]+.._31[6] };
7890 : _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7891 : _36 = _34 + _35;
7892 : // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7893 : // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7894 : _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7895 : _38 = _36 + _37;
7896 : // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7897 : // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7898 : _39 = _30 + _38;
7899 : _50 = _31 + _39;
7900 : _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7901 : MEM <vector(8) int> [(int *)&D.2044] = _39;
7902 : MEM <vector(8) int> [(int *)&D.2042] = _51; */
7903 512 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7904 512 : optab optab = optab_for_tree_code (code, vectype, optab_default);
7905 512 : if (!optab || !can_implement_p (optab, vec_mode))
7906 0 : goto fail;
7907 :
7908 512 : int units_log2 = scan_store_can_perm_p (vectype, *init);
7909 512 : if (units_log2 == -1)
7910 0 : goto fail;
7911 :
7912 : return true;
7913 : }
7914 :
7915 :
7916 : /* Function vectorizable_scan_store.
7917 :
7918 : Helper of vectorizable_score, arguments like on vectorizable_store.
7919 : Handle only the transformation, checking is done in check_scan_store. */
7920 :
7921 : static bool
7922 512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
7923 : slp_tree slp_node, gimple_stmt_iterator *gsi)
7924 : {
7925 512 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7926 512 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7927 512 : tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
7928 512 : tree vectype = SLP_TREE_VECTYPE (slp_node);
7929 :
7930 512 : if (dump_enabled_p ())
7931 492 : dump_printf_loc (MSG_NOTE, vect_location,
7932 : "transform scan store.\n");
7933 :
7934 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7935 512 : tree rhs = gimple_assign_rhs1 (stmt);
7936 512 : gcc_assert (TREE_CODE (rhs) == SSA_NAME);
7937 :
7938 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7939 512 : bool inscan_var_store
7940 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7941 :
7942 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7943 : {
7944 126 : use_operand_p use_p;
7945 126 : imm_use_iterator iter;
7946 126 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7947 : {
7948 126 : gimple *use_stmt = USE_STMT (use_p);
7949 126 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7950 0 : continue;
7951 126 : rhs = gimple_assign_lhs (use_stmt);
7952 126 : break;
7953 126 : }
7954 : }
7955 :
7956 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7957 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7958 512 : if (code == POINTER_PLUS_EXPR)
7959 0 : code = PLUS_EXPR;
7960 512 : gcc_assert (TREE_CODE_LENGTH (code) == binary_op
7961 : && commutative_tree_code (code));
7962 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7963 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7964 512 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
7965 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7966 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7967 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7968 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7969 512 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7970 512 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7971 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7972 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7973 :
7974 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7975 : {
7976 436 : std::swap (rhs1, rhs2);
7977 436 : std::swap (var1, var2);
7978 436 : std::swap (load1_dr_info, load2_dr_info);
7979 : }
7980 :
7981 512 : tree *init = loop_vinfo->scan_map->get (var1);
7982 512 : gcc_assert (init);
7983 :
7984 512 : unsigned HOST_WIDE_INT nunits;
7985 512 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7986 : gcc_unreachable ();
7987 512 : auto_vec<enum scan_store_kind, 16> use_whole_vector;
7988 512 : int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
7989 512 : gcc_assert (units_log2 > 0);
7990 512 : auto_vec<tree, 16> perms;
7991 512 : perms.quick_grow (units_log2 + 1);
7992 512 : tree zero_vec = NULL_TREE, masktype = NULL_TREE;
7993 2904 : for (int i = 0; i <= units_log2; ++i)
7994 : {
7995 1880 : unsigned HOST_WIDE_INT j, k;
7996 1880 : vec_perm_builder sel (nunits, nunits, 1);
7997 1880 : sel.quick_grow (nunits);
7998 1880 : if (i == units_log2)
7999 4864 : for (j = 0; j < nunits; ++j)
8000 4352 : sel[j] = nunits - 1;
8001 : else
8002 : {
8003 5208 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
8004 3840 : sel[j] = j;
8005 13208 : for (k = 0; j < nunits; ++j, ++k)
8006 11840 : sel[j] = nunits + k;
8007 : }
8008 3248 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
8009 1880 : if (!use_whole_vector.is_empty ()
8010 0 : && use_whole_vector[i] != scan_store_kind_perm)
8011 : {
8012 0 : if (zero_vec == NULL_TREE)
8013 0 : zero_vec = build_zero_cst (vectype);
8014 0 : if (masktype == NULL_TREE
8015 0 : && use_whole_vector[i] == scan_store_kind_lshift_cond)
8016 0 : masktype = truth_type_for (vectype);
8017 0 : perms[i] = vect_gen_perm_mask_any (vectype, indices);
8018 : }
8019 : else
8020 1880 : perms[i] = vect_gen_perm_mask_checked (vectype, indices);
8021 1880 : }
8022 :
8023 512 : tree vec_oprnd1 = NULL_TREE;
8024 512 : tree vec_oprnd2 = NULL_TREE;
8025 512 : tree vec_oprnd3 = NULL_TREE;
8026 512 : tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
8027 512 : tree dataref_offset = build_int_cst (ref_type, 0);
8028 512 : tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
8029 512 : tree ldataref_ptr = NULL_TREE;
8030 512 : tree orig = NULL_TREE;
8031 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
8032 126 : ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
8033 : /* The initialization is invariant. */
8034 512 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
8035 512 : auto_vec<tree> vec_oprnds2;
8036 512 : auto_vec<tree> vec_oprnds3;
8037 512 : if (ldataref_ptr == NULL)
8038 : {
8039 : /* We want to lookup the vector operands of the reduction, not those
8040 : of the store - for SLP we have to use the proper SLP node for the
8041 : lookup, which should be the single child of the scan store. */
8042 386 : vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
8043 : rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
8044 : /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
8045 : them in SLP child order. So we have to swap here with logic
8046 : similar to above. */
8047 386 : stmt_vec_info load
8048 386 : = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
8049 386 : (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
8050 386 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
8051 386 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
8052 386 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
8053 820 : for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
8054 494 : std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
8055 : }
8056 : else
8057 126 : vect_get_vec_defs (vinfo, slp_node,
8058 : rhs2, &vec_oprnds3);
8059 1248 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8060 : {
8061 736 : if (ldataref_ptr == NULL)
8062 554 : vec_oprnd2 = vec_oprnds2[j];
8063 736 : vec_oprnd3 = vec_oprnds3[j];
8064 736 : if (j == 0)
8065 : orig = vec_oprnd3;
8066 224 : else if (!inscan_var_store)
8067 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8068 :
8069 736 : if (ldataref_ptr)
8070 : {
8071 182 : vec_oprnd2 = make_ssa_name (vectype);
8072 182 : tree data_ref = fold_build2 (MEM_REF, vectype,
8073 : unshare_expr (ldataref_ptr),
8074 : dataref_offset);
8075 182 : vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
8076 182 : gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
8077 182 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8078 : }
8079 :
8080 736 : tree v = vec_oprnd2;
8081 3068 : for (int i = 0; i < units_log2; ++i)
8082 : {
8083 2332 : tree new_temp = make_ssa_name (vectype);
8084 2332 : gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
8085 : (zero_vec
8086 0 : && (use_whole_vector[i]
8087 0 : != scan_store_kind_perm))
8088 : ? zero_vec : vec_oprnd1, v,
8089 2332 : perms[i]);
8090 2332 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8091 :
8092 2332 : if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
8093 : {
8094 : /* Whole vector shift shifted in zero bits, but if *init
8095 : is not initializer_zerop, we need to replace those elements
8096 : with elements from vec_oprnd1. */
8097 0 : tree_vector_builder vb (masktype, nunits, 1);
8098 0 : for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
8099 0 : vb.quick_push (k < (HOST_WIDE_INT_1U << i)
8100 : ? boolean_false_node : boolean_true_node);
8101 :
8102 0 : tree new_temp2 = make_ssa_name (vectype);
8103 0 : g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
8104 : new_temp, vec_oprnd1);
8105 0 : vect_finish_stmt_generation (vinfo, stmt_info,
8106 : g, gsi);
8107 0 : new_temp = new_temp2;
8108 0 : }
8109 :
8110 : /* For exclusive scan, perform the perms[i] permutation once
8111 : more. */
8112 2332 : if (i == 0
8113 1100 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
8114 728 : && v == vec_oprnd2)
8115 : {
8116 364 : v = new_temp;
8117 364 : --i;
8118 364 : continue;
8119 : }
8120 :
8121 1968 : tree new_temp2 = make_ssa_name (vectype);
8122 1968 : g = gimple_build_assign (new_temp2, code, v, new_temp);
8123 1968 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8124 :
8125 1968 : v = new_temp2;
8126 : }
8127 :
8128 736 : tree new_temp = make_ssa_name (vectype);
8129 736 : gimple *g = gimple_build_assign (new_temp, code, orig, v);
8130 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8131 :
8132 736 : tree last_perm_arg = new_temp;
8133 : /* For exclusive scan, new_temp computed above is the exclusive scan
8134 : prefix sum. Turn it into inclusive prefix sum for the broadcast
8135 : of the last element into orig. */
8136 736 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
8137 : {
8138 364 : last_perm_arg = make_ssa_name (vectype);
8139 364 : g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
8140 364 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8141 : }
8142 :
8143 736 : orig = make_ssa_name (vectype);
8144 2208 : g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
8145 736 : last_perm_arg, perms[units_log2]);
8146 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8147 :
8148 736 : if (!inscan_var_store)
8149 : {
8150 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8151 : unshare_expr (dataref_ptr),
8152 : dataref_offset);
8153 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8154 368 : g = gimple_build_assign (data_ref, new_temp);
8155 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8156 : }
8157 : }
8158 :
8159 512 : if (inscan_var_store)
8160 624 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8161 : {
8162 368 : if (j != 0)
8163 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8164 :
8165 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8166 : unshare_expr (dataref_ptr),
8167 : dataref_offset);
8168 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8169 368 : gimple *g = gimple_build_assign (data_ref, orig);
8170 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8171 : }
8172 512 : return true;
8173 512 : }
8174 :
8175 :
8176 : /* Function vectorizable_store.
8177 :
8178 : Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8179 : that can be vectorized.
8180 : If COST_VEC is passed, calculate costs but don't change anything,
8181 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
8182 : it, and insert it at GSI.
8183 : Return true if STMT_INFO is vectorizable in this way. */
8184 :
8185 : static bool
8186 2167633 : vectorizable_store (vec_info *vinfo,
8187 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8188 : slp_tree slp_node,
8189 : stmt_vector_for_cost *cost_vec)
8190 : {
8191 2167633 : tree data_ref;
8192 2167633 : tree vec_oprnd = NULL_TREE;
8193 2167633 : tree elem_type;
8194 2167633 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
8195 2167633 : class loop *loop = NULL;
8196 2167633 : machine_mode vec_mode;
8197 2167633 : tree dummy;
8198 2167633 : enum vect_def_type rhs_dt = vect_unknown_def_type;
8199 2167633 : enum vect_def_type mask_dt = vect_unknown_def_type;
8200 2167633 : tree dataref_ptr = NULL_TREE;
8201 2167633 : tree dataref_offset = NULL_TREE;
8202 2167633 : int j;
8203 2167633 : stmt_vec_info first_stmt_info;
8204 2167633 : bool grouped_store;
8205 2167633 : unsigned int group_size, i;
8206 2167633 : unsigned int vec_num;
8207 2167633 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
8208 2167633 : tree aggr_type;
8209 2167633 : poly_uint64 vf;
8210 2167633 : vec_load_store_type vls_type;
8211 2167633 : tree ref_type;
8212 :
8213 2167633 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8214 : return false;
8215 :
8216 2167633 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8217 251571 : && cost_vec)
8218 : return false;
8219 :
8220 : /* Is vectorizable store? */
8221 :
8222 1916062 : tree mask_vectype = NULL_TREE;
8223 1916062 : slp_tree mask_node = NULL;
8224 1916062 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
8225 : {
8226 1840187 : tree scalar_dest = gimple_assign_lhs (assign);
8227 1840187 : if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
8228 1840187 : && is_pattern_stmt_p (stmt_info))
8229 1677 : scalar_dest = TREE_OPERAND (scalar_dest, 0);
8230 1840187 : if (TREE_CODE (scalar_dest) != ARRAY_REF
8231 1840187 : && TREE_CODE (scalar_dest) != BIT_FIELD_REF
8232 : && TREE_CODE (scalar_dest) != INDIRECT_REF
8233 : && TREE_CODE (scalar_dest) != COMPONENT_REF
8234 : && TREE_CODE (scalar_dest) != IMAGPART_EXPR
8235 : && TREE_CODE (scalar_dest) != REALPART_EXPR
8236 : && TREE_CODE (scalar_dest) != MEM_REF)
8237 : return false;
8238 : }
8239 : else
8240 : {
8241 75875 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
8242 13965 : if (!call || !gimple_call_internal_p (call))
8243 : return false;
8244 :
8245 8269 : internal_fn ifn = gimple_call_internal_fn (call);
8246 8269 : if (!internal_store_fn_p (ifn))
8247 : return false;
8248 :
8249 1883 : int mask_index = internal_fn_mask_index (ifn);
8250 1883 : if (mask_index >= 0)
8251 1883 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
8252 1883 : if (mask_index >= 0
8253 1883 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
8254 : &mask_node, &mask_dt,
8255 : &mask_vectype))
8256 : return false;
8257 : }
8258 :
8259 1387956 : tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
8260 1387956 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8261 :
8262 1387956 : if (loop_vinfo)
8263 : {
8264 235905 : loop = LOOP_VINFO_LOOP (loop_vinfo);
8265 235905 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8266 : }
8267 : else
8268 : vf = 1;
8269 1387956 : vec_num = vect_get_num_copies (vinfo, slp_node);
8270 :
8271 : /* FORNOW. This restriction should be relaxed. */
8272 1387956 : if (loop
8273 1388243 : && nested_in_vect_loop_p (loop, stmt_info)
8274 1388251 : && vec_num > 1)
8275 : {
8276 8 : if (dump_enabled_p ())
8277 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8278 : "multiple types in nested loop.\n");
8279 : return false;
8280 : }
8281 :
8282 1387948 : slp_tree op_node;
8283 1387948 : if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
8284 : &op_node, &rhs_dt, &rhs_vectype, &vls_type))
8285 : return false;
8286 :
8287 1387924 : elem_type = TREE_TYPE (vectype);
8288 1387924 : vec_mode = TYPE_MODE (vectype);
8289 :
8290 1387924 : if (!STMT_VINFO_DATA_REF (stmt_info))
8291 : return false;
8292 :
8293 1387924 : vect_load_store_data _ls_data{};
8294 1387924 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
8295 1387924 : if (cost_vec
8296 1387924 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
8297 : vls_type, &_ls_data))
8298 : return false;
8299 : /* Temporary aliases to analysis data, should not be modified through
8300 : these. */
8301 1387408 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
8302 1387408 : const dr_alignment_support alignment_support_scheme
8303 : = ls.alignment_support_scheme;
8304 1387408 : const int misalignment = ls.misalignment;
8305 1387408 : const poly_int64 poffset = ls.poffset;
8306 :
8307 1387408 : if (slp_node->ldst_lanes
8308 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
8309 : {
8310 0 : if (dump_enabled_p ())
8311 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8312 : "discovered store-lane but cannot use it.\n");
8313 : return false;
8314 : }
8315 :
8316 1387408 : if (mask_node)
8317 : {
8318 1793 : if (memory_access_type == VMAT_CONTIGUOUS)
8319 : {
8320 616 : if (!VECTOR_MODE_P (vec_mode)
8321 3070 : || !can_vec_mask_load_store_p (vec_mode,
8322 1535 : TYPE_MODE (mask_vectype), false))
8323 : return false;
8324 : }
8325 258 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
8326 258 : && (!mat_gather_scatter_p (memory_access_type)
8327 242 : || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
8328 170 : && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
8329 : {
8330 16 : if (dump_enabled_p ())
8331 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8332 : "unsupported access type for masked store.\n");
8333 : return false;
8334 : }
8335 242 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
8336 : {
8337 72 : if (dump_enabled_p ())
8338 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8339 : "unsupported masked emulated scatter.\n");
8340 : return false;
8341 : }
8342 : }
8343 : else
8344 : {
8345 : /* FORNOW. In some cases can vectorize even if data-type not supported
8346 : (e.g. - array initialization with 0). */
8347 1385615 : if (!can_implement_p (mov_optab, vec_mode))
8348 : return false;
8349 : }
8350 :
8351 1387206 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
8352 1387206 : grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
8353 2559598 : && !mat_gather_scatter_p (memory_access_type));
8354 1172392 : if (grouped_store)
8355 : {
8356 1172392 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8357 1172392 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8358 1172392 : group_size = DR_GROUP_SIZE (first_stmt_info);
8359 : }
8360 : else
8361 : {
8362 1387206 : first_stmt_info = stmt_info;
8363 1387206 : first_dr_info = dr_info;
8364 : group_size = 1;
8365 : }
8366 :
8367 1387206 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
8368 : {
8369 1076 : if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
8370 : mask_node, memory_access_type))
8371 : return false;
8372 : }
8373 :
8374 1936950 : bool costing_p = cost_vec;
8375 1386438 : if (costing_p) /* transformation not required. */
8376 : {
8377 836694 : if (loop_vinfo
8378 169382 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
8379 79482 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
8380 : vls_type, group_size, &ls,
8381 : mask_node);
8382 :
8383 836694 : if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
8384 836694 : || (mask_node
8385 1051 : && !vect_maybe_update_slp_op_vectype (mask_node,
8386 : mask_vectype)))
8387 : {
8388 0 : if (dump_enabled_p ())
8389 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8390 : "incompatible vector types for invariants\n");
8391 : return false;
8392 : }
8393 :
8394 836694 : if (dump_enabled_p ()
8395 : && memory_access_type != VMAT_ELEMENTWISE
8396 15867 : && memory_access_type != VMAT_STRIDED_SLP
8397 15200 : && memory_access_type != VMAT_INVARIANT
8398 851894 : && alignment_support_scheme != dr_aligned)
8399 5366 : dump_printf_loc (MSG_NOTE, vect_location,
8400 : "Vectorizing an unaligned access.\n");
8401 : }
8402 :
8403 : /* Transform. */
8404 :
8405 : if (!costing_p)
8406 550512 : ensure_base_align (dr_info);
8407 :
8408 1387206 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
8409 : {
8410 1024 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
8411 1024 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8412 1024 : if (costing_p)
8413 : {
8414 512 : unsigned int inside_cost = 0, prologue_cost = 0;
8415 512 : if (vls_type == VLS_STORE_INVARIANT)
8416 0 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8417 : slp_node, 0, vect_prologue);
8418 512 : vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
8419 : alignment_support_scheme, misalignment,
8420 : &inside_cost, cost_vec);
8421 :
8422 512 : if (dump_enabled_p ())
8423 492 : dump_printf_loc (MSG_NOTE, vect_location,
8424 : "vect_model_store_cost: inside_cost = %d, "
8425 : "prologue_cost = %d .\n",
8426 : inside_cost, prologue_cost);
8427 :
8428 512 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8429 512 : slp_node->data = new vect_load_store_data (std::move (ls));
8430 :
8431 512 : return true;
8432 : }
8433 512 : return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
8434 : }
8435 :
8436 : /* FORNOW */
8437 1386182 : gcc_assert (!grouped_store
8438 : || !loop
8439 : || !nested_in_vect_loop_p (loop, stmt_info));
8440 :
8441 1386182 : grouped_store = false;
8442 1386182 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
8443 1386182 : gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
8444 : || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
8445 1386182 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8446 :
8447 1386182 : ref_type = get_group_alias_ptr_type (first_stmt_info);
8448 :
8449 1386182 : if (!costing_p && dump_enabled_p ())
8450 12812 : dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
8451 :
8452 1386182 : if (memory_access_type == VMAT_ELEMENTWISE
8453 1386182 : || memory_access_type == VMAT_STRIDED_SLP)
8454 : {
8455 29784 : unsigned inside_cost = 0, prologue_cost = 0;
8456 29784 : gimple_stmt_iterator incr_gsi;
8457 29784 : bool insert_after;
8458 29784 : tree offvar = NULL_TREE;
8459 29784 : tree ivstep;
8460 29784 : tree running_off;
8461 29784 : tree stride_base, stride_step, alias_off;
8462 29784 : tree vec_oprnd = NULL_TREE;
8463 29784 : tree dr_offset;
8464 : /* Checked by get_load_store_type. */
8465 29784 : unsigned int const_nunits = nunits.to_constant ();
8466 :
8467 29784 : gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
8468 29784 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
8469 :
8470 29784 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
8471 29784 : stride_base
8472 29784 : = fold_build_pointer_plus
8473 : (DR_BASE_ADDRESS (first_dr_info->dr),
8474 : size_binop (PLUS_EXPR,
8475 : convert_to_ptrofftype (dr_offset),
8476 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
8477 29784 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
8478 :
8479 : /* For a store with loop-invariant (but other than power-of-2)
8480 : stride (i.e. not a grouped access) like so:
8481 :
8482 : for (i = 0; i < n; i += stride)
8483 : array[i] = ...;
8484 :
8485 : we generate a new induction variable and new stores from
8486 : the components of the (vectorized) rhs:
8487 :
8488 : for (j = 0; ; j += VF*stride)
8489 : vectemp = ...;
8490 : tmp1 = vectemp[0];
8491 : array[j] = tmp1;
8492 : tmp2 = vectemp[1];
8493 : array[j + stride] = tmp2;
8494 : ...
8495 : */
8496 :
8497 : /* ??? Modify local copies of alignment_support_scheme and
8498 : misalignment, but this part of analysis should be done
8499 : earlier and remembered, likewise the chosen load mode. */
8500 29784 : const dr_alignment_support tem = alignment_support_scheme;
8501 29784 : dr_alignment_support alignment_support_scheme = tem;
8502 29784 : const int tem2 = misalignment;
8503 29784 : int misalignment = tem2;
8504 :
8505 29784 : unsigned nstores = const_nunits;
8506 29784 : unsigned lnel = 1;
8507 29784 : tree ltype = elem_type;
8508 29784 : tree lvectype = vectype;
8509 29784 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
8510 29784 : if (n == const_nunits)
8511 : {
8512 2978 : int mis_align = dr_misalignment (first_dr_info, vectype);
8513 : /* With VF > 1 we advance the DR by step, if that is constant
8514 : and only aligned when performed VF times, DR alignment
8515 : analysis can analyze this as aligned since it assumes
8516 : contiguous accesses. But that is not how we code generate
8517 : here, so adjust for this. */
8518 2978 : if (maybe_gt (vf, 1u)
8519 4514 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8520 4279 : DR_TARGET_ALIGNMENT (first_dr_info)))
8521 235 : mis_align = -1;
8522 2978 : dr_alignment_support dr_align
8523 2978 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
8524 : mis_align);
8525 2978 : if (dr_align == dr_aligned
8526 2978 : || dr_align == dr_unaligned_supported)
8527 : {
8528 29784 : nstores = 1;
8529 29784 : lnel = const_nunits;
8530 29784 : ltype = vectype;
8531 29784 : lvectype = vectype;
8532 29784 : alignment_support_scheme = dr_align;
8533 29784 : misalignment = mis_align;
8534 : }
8535 : }
8536 26806 : else if (n > 1)
8537 : {
8538 1969 : nstores = const_nunits / n;
8539 1969 : lnel = n;
8540 1969 : ltype = build_vector_type (elem_type, n);
8541 1969 : lvectype = vectype;
8542 1969 : int mis_align = dr_misalignment (first_dr_info, ltype);
8543 1969 : if (maybe_gt (vf, 1u)
8544 3938 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8545 3292 : DR_TARGET_ALIGNMENT (first_dr_info)))
8546 646 : mis_align = -1;
8547 1969 : dr_alignment_support dr_align
8548 1969 : = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
8549 : mis_align);
8550 1969 : alignment_support_scheme = dr_align;
8551 1969 : misalignment = mis_align;
8552 :
8553 : /* First check if vec_extract optab doesn't support extraction
8554 : of vector elts directly. */
8555 1969 : scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
8556 1969 : machine_mode vmode;
8557 3938 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8558 1969 : || !related_vector_mode (TYPE_MODE (vectype), elmode,
8559 1969 : n).exists (&vmode)
8560 2141 : || can_vec_extract (TYPE_MODE (vectype), vmode) != vmode
8561 1969 : || !(dr_align == dr_aligned
8562 172 : || dr_align == dr_unaligned_supported))
8563 : {
8564 : /* Try to avoid emitting an extract of vector elements
8565 : by performing the extracts using an integer type of the
8566 : same size, extracting from a vector of those and then
8567 : re-interpreting it as the original vector type if
8568 : supported. */
8569 1797 : unsigned lsize = n * GET_MODE_BITSIZE (elmode);
8570 1797 : unsigned int lnunits = const_nunits / n;
8571 : /* If we can't construct such a vector fall back to
8572 : element extracts from the original vector type and
8573 : element size stores. */
8574 1797 : if (int_mode_for_size (lsize, 0).exists (&elmode)
8575 1797 : && VECTOR_MODE_P (TYPE_MODE (vectype))
8576 1797 : && related_vector_mode (TYPE_MODE (vectype), elmode,
8577 1797 : lnunits).exists (&vmode)
8578 1769 : && can_vec_extract (vmode, elmode).exists ())
8579 : {
8580 1769 : nstores = lnunits;
8581 1769 : lnel = n;
8582 1769 : ltype = build_nonstandard_integer_type (lsize, 1);
8583 1769 : lvectype = build_vector_type (ltype, nstores);
8584 : }
8585 : /* Else fall back to vector extraction anyway.
8586 : Fewer stores are more important than avoiding spilling
8587 : of the vector we extract from. Compared to the
8588 : construction case in vectorizable_load no store-forwarding
8589 : issue exists here for reasonable archs. But only
8590 : if the store is supported. */
8591 28 : else if (!(dr_align == dr_aligned
8592 28 : || dr_align == dr_unaligned_supported))
8593 : {
8594 29784 : nstores = const_nunits;
8595 29784 : lnel = 1;
8596 29784 : ltype = elem_type;
8597 29784 : lvectype = vectype;
8598 : }
8599 : }
8600 : }
8601 :
8602 29784 : if (costing_p)
8603 : {
8604 : /* Record the decomposition type for target access during costing. */
8605 26331 : ls.ls_type = lvectype;
8606 26331 : ls.ls_eltype = ltype;
8607 : }
8608 : else
8609 3453 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
8610 :
8611 29784 : unsigned align;
8612 29784 : if (alignment_support_scheme == dr_aligned)
8613 1263 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
8614 : else
8615 28521 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
8616 : /* Alignment is at most the access size if we do multiple stores. */
8617 29784 : if (nstores > 1)
8618 26806 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
8619 29784 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
8620 29784 : int ncopies = vec_num;
8621 :
8622 29784 : if (!costing_p)
8623 : {
8624 3453 : ivstep = stride_step;
8625 :
8626 3453 : tree increment = fold_convert (TREE_TYPE (ivstep),
8627 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
8628 :
8629 3453 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
8630 : increment);
8631 :
8632 3453 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
8633 :
8634 3453 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
8635 3453 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
8636 3453 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
8637 : else
8638 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
8639 : true, NULL_TREE, true,
8640 : GSI_SAME_STMT);
8641 :
8642 3453 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
8643 : insert_after, &offvar, NULL,
8644 3453 : LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
8645 :
8646 3453 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
8647 : }
8648 :
8649 29784 : alias_off = build_int_cst (ref_type, 0);
8650 29784 : auto_vec<tree> vec_oprnds;
8651 : /* For costing some adjacent vector stores, we'd like to cost with
8652 : the total number of them once instead of cost each one by one. */
8653 29784 : unsigned int n_adjacent_stores = 0;
8654 29784 : running_off = offvar;
8655 29784 : if (!costing_p)
8656 3453 : vect_get_slp_defs (op_node, &vec_oprnds);
8657 29784 : unsigned int group_el = 0;
8658 29784 : unsigned HOST_WIDE_INT elsz
8659 29784 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
8660 70883 : for (j = 0; j < ncopies; j++)
8661 : {
8662 41099 : if (!costing_p)
8663 : {
8664 5347 : vec_oprnd = vec_oprnds[j];
8665 : /* Pun the vector to extract from if necessary. */
8666 5347 : if (lvectype != vectype)
8667 : {
8668 1010 : tree tem = make_ssa_name (lvectype);
8669 1010 : tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
8670 1010 : gimple *pun = gimple_build_assign (tem, cvt);
8671 1010 : vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
8672 1010 : vec_oprnd = tem;
8673 : }
8674 : }
8675 182776 : for (i = 0; i < nstores; i++)
8676 : {
8677 141677 : if (costing_p)
8678 : {
8679 125495 : n_adjacent_stores++;
8680 125495 : continue;
8681 : }
8682 16182 : tree newref, newoff;
8683 16182 : gimple *incr, *assign;
8684 16182 : tree size = TYPE_SIZE (ltype);
8685 : /* Extract the i'th component. */
8686 16182 : tree pos = fold_build2 (MULT_EXPR, bitsizetype,
8687 : bitsize_int (i), size);
8688 16182 : tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
8689 : size, pos);
8690 :
8691 16182 : elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
8692 : GSI_SAME_STMT);
8693 :
8694 16182 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
8695 16182 : group_el * elsz);
8696 16182 : newref = build2 (MEM_REF, ltype, running_off, this_off);
8697 16182 : vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
8698 :
8699 : /* And store it to *running_off. */
8700 16182 : assign = gimple_build_assign (newref, elem);
8701 16182 : vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
8702 :
8703 16182 : group_el += lnel;
8704 16182 : if (group_el == group_size)
8705 : {
8706 14506 : newoff = copy_ssa_name (running_off, NULL);
8707 14506 : incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
8708 : running_off, stride_step);
8709 14506 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
8710 :
8711 14506 : running_off = newoff;
8712 14506 : group_el = 0;
8713 : }
8714 : }
8715 : }
8716 :
8717 29784 : if (costing_p)
8718 : {
8719 26331 : if (n_adjacent_stores > 0)
8720 : {
8721 : /* Take a single lane vector type store as scalar
8722 : store to avoid ICE like 110776. */
8723 26331 : if (VECTOR_TYPE_P (ltype)
8724 26331 : && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
8725 1616 : vect_get_store_cost (vinfo, stmt_info, slp_node,
8726 : n_adjacent_stores, alignment_support_scheme,
8727 : misalignment, &inside_cost, cost_vec);
8728 : else
8729 24715 : inside_cost
8730 24715 : += record_stmt_cost (cost_vec, n_adjacent_stores,
8731 : scalar_store, slp_node, 0, vect_body);
8732 : /* Only need vector deconstruction when there is more
8733 : than one store. */
8734 26331 : if (nstores > 1)
8735 24299 : inside_cost
8736 24299 : += record_stmt_cost (cost_vec, ncopies,
8737 : vec_deconstruct, slp_node, 0, vect_body);
8738 : }
8739 26331 : if (dump_enabled_p ())
8740 667 : dump_printf_loc (MSG_NOTE, vect_location,
8741 : "vect_model_store_cost: inside_cost = %d, "
8742 : "prologue_cost = %d .\n",
8743 : inside_cost, prologue_cost);
8744 :
8745 26331 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8746 26331 : slp_node->data = new vect_load_store_data (std::move (ls));
8747 : }
8748 :
8749 29784 : return true;
8750 29784 : }
8751 :
8752 1356398 : gcc_assert (alignment_support_scheme);
8753 1356398 : vec_loop_masks *loop_masks
8754 204347 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
8755 1356398 : ? &LOOP_VINFO_MASKS (loop_vinfo)
8756 16 : : NULL);
8757 16 : vec_loop_lens *loop_lens
8758 204347 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
8759 : ? &LOOP_VINFO_LENS (loop_vinfo)
8760 0 : : NULL);
8761 :
8762 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8763 : are some difference here. We cannot enable both the lens and masks
8764 : during transform but it is allowed during analysis.
8765 : Shouldn't go with length-based approach if fully masked. */
8766 1356398 : if (cost_vec == NULL)
8767 : /* The cost_vec is NULL during transform. */
8768 546547 : gcc_assert ((!loop_lens || !loop_masks));
8769 :
8770 : /* Targets with store-lane instructions must not require explicit
8771 : realignment. vect_supportable_dr_alignment always returns either
8772 : dr_aligned or dr_unaligned_supported for masked operations. */
8773 1356398 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
8774 : && !mask_node
8775 : && !loop_masks)
8776 : || alignment_support_scheme == dr_aligned
8777 : || alignment_support_scheme == dr_unaligned_supported);
8778 :
8779 1356398 : tree offset = NULL_TREE;
8780 1356398 : if (!known_eq (poffset, 0))
8781 4699 : offset = size_int (poffset);
8782 :
8783 1356398 : tree dr_increment;
8784 1356398 : tree dr_bump;
8785 :
8786 1356398 : tree vec_offset = NULL_TREE;
8787 1356398 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8788 : {
8789 1383 : aggr_type = NULL_TREE;
8790 1383 : dr_increment = NULL_TREE;
8791 1383 : dr_bump = NULL_TREE;
8792 : }
8793 1355015 : else if (mat_gather_scatter_p (memory_access_type))
8794 : {
8795 0 : aggr_type = elem_type;
8796 0 : if (!costing_p)
8797 : {
8798 0 : tree vtype = ls.ls_type ? ls.ls_type : vectype;
8799 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
8800 : ls.strided_offset_vectype,
8801 : loop_vinfo, gsi,
8802 : &dr_increment, &dr_bump,
8803 : &vec_offset);
8804 : }
8805 : }
8806 : else
8807 : {
8808 1355015 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8809 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
8810 : else
8811 : aggr_type = vectype;
8812 1355015 : if (!costing_p)
8813 : {
8814 546073 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
8815 : memory_access_type);
8816 546073 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
8817 : memory_access_type);
8818 : }
8819 : }
8820 :
8821 1356398 : if (loop_vinfo && mask_node && !costing_p)
8822 540 : LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
8823 :
8824 : /* In case the vectorization factor (VF) is bigger than the number
8825 : of elements that we can fit in a vectype (nunits), we have to generate
8826 : more than one vector stmt - i.e - we need to "unroll" the
8827 : vector stmt by a factor VF/nunits. */
8828 :
8829 1356398 : auto_vec<tree> dr_chain (group_size);
8830 1356398 : auto_vec<tree> vec_masks;
8831 1356398 : tree vec_mask = NULL;
8832 1356398 : auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
8833 6106063 : for (i = 0; i < group_size; i++)
8834 3393267 : gvec_oprnds.quick_push (new auto_vec<tree> ());
8835 :
8836 1356398 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8837 : {
8838 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
8839 :
8840 0 : if (costing_p)
8841 : /* Update all incoming store operand nodes, the general handling
8842 : above only handles the mask and the first store operand node. */
8843 0 : for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
8844 0 : if (child != mask_node
8845 0 : && !vect_maybe_update_slp_op_vectype (child, vectype))
8846 : {
8847 0 : if (dump_enabled_p ())
8848 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8849 : "incompatible vector types for invariants\n");
8850 : return false;
8851 : }
8852 0 : unsigned inside_cost = 0, prologue_cost = 0;
8853 : /* For costing some adjacent vector stores, we'd like to cost with
8854 : the total number of them once instead of cost each one by one. */
8855 0 : unsigned int n_adjacent_stores = 0;
8856 0 : int ncopies = vec_num / group_size;
8857 0 : for (j = 0; j < ncopies; j++)
8858 : {
8859 0 : if (j == 0)
8860 : {
8861 0 : if (!costing_p)
8862 : {
8863 0 : if (mask_node)
8864 : {
8865 0 : vect_get_slp_defs (mask_node, &vec_masks);
8866 0 : vec_mask = vec_masks[0];
8867 : }
8868 0 : dataref_ptr
8869 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8870 : aggr_type, NULL, offset, &dummy,
8871 : gsi, NULL, false, dr_increment);
8872 : }
8873 : }
8874 0 : else if (!costing_p)
8875 : {
8876 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8877 0 : if (mask_node)
8878 0 : vec_mask = vec_masks[j];
8879 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
8880 : stmt_info, dr_bump);
8881 : }
8882 :
8883 0 : if (costing_p)
8884 : {
8885 0 : n_adjacent_stores += group_size;
8886 0 : continue;
8887 : }
8888 :
8889 : /* Get an array into which we can store the individual vectors. */
8890 0 : tree vec_array = create_vector_array (vectype, group_size);
8891 :
8892 : /* Invalidate the current contents of VEC_ARRAY. This should
8893 : become an RTL clobber too, which prevents the vector registers
8894 : from being upward-exposed. */
8895 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8896 :
8897 : /* Store the individual vectors into the array. */
8898 0 : for (i = 0; i < group_size; i++)
8899 : {
8900 0 : slp_tree child;
8901 0 : if (i == 0 || !mask_node)
8902 0 : child = SLP_TREE_CHILDREN (slp_node)[i];
8903 : else
8904 0 : child = SLP_TREE_CHILDREN (slp_node)[i + 1];
8905 0 : vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
8906 0 : write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
8907 : i);
8908 : }
8909 :
8910 0 : tree final_mask = NULL;
8911 0 : tree final_len = NULL;
8912 0 : tree bias = NULL;
8913 0 : if (loop_masks)
8914 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
8915 : ncopies, vectype, j);
8916 0 : if (vec_mask)
8917 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
8918 : vec_mask, gsi);
8919 :
8920 0 : if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
8921 : {
8922 0 : if (loop_lens)
8923 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
8924 : ncopies, vectype, j, 1, true);
8925 : else
8926 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8927 0 : signed char biasval
8928 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
8929 0 : bias = build_int_cst (intQI_type_node, biasval);
8930 0 : if (!final_mask)
8931 : {
8932 0 : mask_vectype = truth_type_for (vectype);
8933 0 : final_mask = build_minus_one_cst (mask_vectype);
8934 : }
8935 : }
8936 :
8937 0 : gcall *call;
8938 0 : if (final_len && final_mask)
8939 : {
8940 : /* Emit:
8941 : MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8942 : LEN, BIAS, VEC_ARRAY). */
8943 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8944 0 : tree alias_ptr = build_int_cst (ref_type, align);
8945 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
8946 : dataref_ptr, alias_ptr,
8947 : final_mask, final_len, bias,
8948 : vec_array);
8949 : }
8950 0 : else if (final_mask)
8951 : {
8952 : /* Emit:
8953 : MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8954 : VEC_ARRAY). */
8955 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8956 0 : tree alias_ptr = build_int_cst (ref_type, align);
8957 0 : call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
8958 : dataref_ptr, alias_ptr,
8959 : final_mask, vec_array);
8960 : }
8961 : else
8962 : {
8963 : /* Emit:
8964 : MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8965 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8966 0 : call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
8967 0 : gimple_call_set_lhs (call, data_ref);
8968 : }
8969 0 : gimple_call_set_nothrow (call, true);
8970 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
8971 :
8972 : /* Record that VEC_ARRAY is now dead. */
8973 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8974 : }
8975 :
8976 0 : if (costing_p)
8977 : {
8978 0 : if (n_adjacent_stores > 0)
8979 0 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
8980 : alignment_support_scheme, misalignment,
8981 : &inside_cost, cost_vec);
8982 0 : if (dump_enabled_p ())
8983 0 : dump_printf_loc (MSG_NOTE, vect_location,
8984 : "vect_model_store_cost: inside_cost = %d, "
8985 : "prologue_cost = %d .\n",
8986 : inside_cost, prologue_cost);
8987 :
8988 0 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8989 0 : slp_node->data = new vect_load_store_data (std::move (ls));
8990 : }
8991 :
8992 : return true;
8993 : }
8994 :
8995 1356398 : if (mat_gather_scatter_p (memory_access_type))
8996 : {
8997 1383 : gcc_assert (!grouped_store || ls.ls_type);
8998 1383 : if (ls.ls_type)
8999 0 : vectype = ls.ls_type;
9000 1383 : auto_vec<tree> vec_offsets;
9001 1383 : unsigned int inside_cost = 0, prologue_cost = 0;
9002 1383 : int num_stmts = vec_num;
9003 3023 : for (j = 0; j < num_stmts; j++)
9004 : {
9005 1640 : gimple *new_stmt;
9006 1640 : if (j == 0)
9007 : {
9008 1383 : if (costing_p && vls_type == VLS_STORE_INVARIANT)
9009 148 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
9010 : slp_node, 0, vect_prologue);
9011 : else if (!costing_p)
9012 : {
9013 : /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
9014 : DR_CHAIN is of size 1. */
9015 474 : gcc_assert (group_size == 1);
9016 474 : vect_get_slp_defs (op_node, gvec_oprnds[0]);
9017 474 : if (mask_node)
9018 70 : vect_get_slp_defs (mask_node, &vec_masks);
9019 :
9020 474 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9021 474 : vect_get_gather_scatter_ops (loop, slp_node,
9022 : &dataref_ptr, &vec_offsets);
9023 : else
9024 0 : dataref_ptr
9025 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
9026 : aggr_type, NULL, offset,
9027 : &dummy, gsi, NULL, false,
9028 : dr_increment);
9029 : }
9030 : }
9031 257 : else if (!costing_p)
9032 : {
9033 35 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
9034 35 : if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9035 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
9036 : gsi, stmt_info, dr_bump);
9037 : }
9038 :
9039 2297 : new_stmt = NULL;
9040 657 : if (!costing_p)
9041 : {
9042 509 : vec_oprnd = (*gvec_oprnds[0])[j];
9043 509 : if (mask_node)
9044 90 : vec_mask = vec_masks[j];
9045 : /* We should have caught mismatched types earlier. */
9046 509 : gcc_assert (ls.ls_type
9047 : || useless_type_conversion_p
9048 : (vectype, TREE_TYPE (vec_oprnd)));
9049 : }
9050 509 : tree final_mask = NULL_TREE;
9051 2149 : tree final_len = NULL_TREE;
9052 2149 : tree bias = NULL_TREE;
9053 509 : if (!costing_p)
9054 : {
9055 509 : if (loop_masks)
9056 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi,
9057 : loop_masks, num_stmts,
9058 : vectype, j);
9059 509 : if (vec_mask)
9060 90 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
9061 : final_mask, vec_mask, gsi);
9062 : }
9063 :
9064 1640 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
9065 1640 : tree alias_align_ptr = build_int_cst (ref_type, align);
9066 1640 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
9067 : {
9068 0 : if (costing_p)
9069 : {
9070 0 : if (ls.supported_offset_vectype
9071 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
9072 : vec_offset))
9073 0 : inside_cost
9074 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9075 : slp_node, 0, vect_body);
9076 0 : if (ls.supported_scale)
9077 0 : inside_cost
9078 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9079 : slp_node, 0, vect_body);
9080 :
9081 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9082 0 : inside_cost
9083 0 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9084 : slp_node, 0, vect_body);
9085 0 : continue;
9086 0 : }
9087 :
9088 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9089 0 : vec_offset = vec_offsets[j];
9090 :
9091 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9092 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
9093 :
9094 : /* Perform the offset conversion and scaling if necessary. */
9095 0 : if (!strided
9096 0 : && (ls.supported_offset_vectype || ls.supported_scale))
9097 : {
9098 0 : gimple_seq stmts = NULL;
9099 0 : if (ls.supported_offset_vectype)
9100 0 : vec_offset = gimple_convert
9101 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
9102 0 : if (ls.supported_scale)
9103 : {
9104 : /* Only scale the vec_offset if we haven't already. */
9105 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
9106 0 : || j == 0)
9107 : {
9108 0 : tree mult_cst = build_int_cst
9109 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
9110 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
9111 0 : tree mult = build_vector_from_val
9112 0 : (TREE_TYPE (vec_offset), mult_cst);
9113 0 : vec_offset = gimple_build
9114 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
9115 : vec_offset, mult);
9116 : }
9117 0 : scale = size_int (ls.supported_scale);
9118 : }
9119 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9120 : }
9121 :
9122 0 : if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
9123 : {
9124 0 : if (loop_lens)
9125 0 : final_len = vect_get_loop_len (loop_vinfo, gsi,
9126 : loop_lens, num_stmts,
9127 : vectype, j, 1, true);
9128 : else
9129 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9130 :
9131 0 : signed char biasval
9132 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9133 0 : bias = build_int_cst (intQI_type_node, biasval);
9134 0 : if (!final_mask)
9135 : {
9136 0 : mask_vectype = truth_type_for (vectype);
9137 0 : final_mask = build_minus_one_cst (mask_vectype);
9138 : }
9139 : }
9140 :
9141 0 : if (ls.ls_type)
9142 : {
9143 0 : gimple *conv_stmt
9144 0 : = gimple_build_assign (make_ssa_name (vectype),
9145 : VIEW_CONVERT_EXPR,
9146 : build1 (VIEW_CONVERT_EXPR, vectype,
9147 : vec_oprnd));
9148 0 : vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
9149 : gsi);
9150 0 : vec_oprnd = gimple_get_lhs (conv_stmt);
9151 : }
9152 :
9153 0 : gcall *call;
9154 0 : if (final_len && final_mask)
9155 : {
9156 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
9157 0 : call = gimple_build_call_internal (
9158 : IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
9159 : alias_align_ptr,
9160 : vec_offset, scale, vec_oprnd, final_mask, final_len,
9161 : bias);
9162 : else
9163 : /* Non-vector offset indicates that prefer to take
9164 : MASK_LEN_STRIDED_STORE instead of the
9165 : IFN_MASK_SCATTER_STORE with direct stride arg.
9166 : Similar to the gather case we have checked the
9167 : alignment for a scatter already and assume
9168 : that the strided store has the same requirements. */
9169 0 : call = gimple_build_call_internal (
9170 : IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
9171 : vec_offset, vec_oprnd, final_mask, final_len, bias);
9172 : }
9173 0 : else if (final_mask)
9174 0 : call = gimple_build_call_internal
9175 0 : (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
9176 : alias_align_ptr,
9177 : vec_offset, scale, vec_oprnd, final_mask);
9178 : else
9179 0 : call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
9180 : dataref_ptr,
9181 : alias_align_ptr,
9182 : vec_offset,
9183 : scale, vec_oprnd);
9184 0 : gimple_call_set_nothrow (call, true);
9185 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9186 0 : new_stmt = call;
9187 : }
9188 1640 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
9189 : {
9190 : /* The builtin decls path for scatter is legacy, x86 only. */
9191 331 : gcc_assert (nunits.is_constant ()
9192 : && (!final_mask
9193 : || SCALAR_INT_MODE_P
9194 : (TYPE_MODE (TREE_TYPE (final_mask)))));
9195 331 : if (costing_p)
9196 : {
9197 200 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9198 200 : inside_cost
9199 200 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9200 : slp_node, 0, vect_body);
9201 200 : continue;
9202 200 : }
9203 :
9204 131 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9205 131 : poly_uint64 offset_nunits
9206 131 : = TYPE_VECTOR_SUBPARTS (offset_vectype);
9207 131 : if (known_eq (nunits, offset_nunits))
9208 : {
9209 55 : new_stmt = vect_build_one_scatter_store_call
9210 110 : (vinfo, stmt_info, slp_node, gsi,
9211 55 : ls.gs.decl, dataref_ptr, vec_offsets[j],
9212 : vec_oprnd, final_mask);
9213 55 : vect_finish_stmt_generation (vinfo, stmt_info,
9214 : new_stmt, gsi);
9215 : }
9216 76 : else if (known_eq (nunits, offset_nunits * 2))
9217 : {
9218 : /* We have a offset vector with half the number of
9219 : lanes but the builtins will store full vectype
9220 : data from the lower lanes. */
9221 30 : new_stmt = vect_build_one_scatter_store_call
9222 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9223 30 : dataref_ptr, vec_offsets[2 * j],
9224 : vec_oprnd, final_mask);
9225 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9226 : new_stmt, gsi);
9227 30 : int count = nunits.to_constant ();
9228 30 : vec_perm_builder sel (count, count, 1);
9229 30 : sel.quick_grow (count);
9230 412 : for (int i = 0; i < count; ++i)
9231 352 : sel[i] = i | (count / 2);
9232 30 : vec_perm_indices indices (sel, 2, count);
9233 30 : tree perm_mask
9234 30 : = vect_gen_perm_mask_checked (vectype, indices);
9235 30 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
9236 : vec_oprnd, vec_oprnd,
9237 : perm_mask);
9238 30 : vec_oprnd = make_ssa_name (vectype);
9239 30 : gimple_set_lhs (new_stmt, vec_oprnd);
9240 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9241 : new_stmt, gsi);
9242 30 : if (final_mask)
9243 : {
9244 20 : new_stmt = gimple_build_assign (NULL_TREE,
9245 : VEC_UNPACK_HI_EXPR,
9246 : final_mask);
9247 20 : final_mask = make_ssa_name
9248 20 : (truth_type_for (offset_vectype));
9249 20 : gimple_set_lhs (new_stmt, final_mask);
9250 20 : vect_finish_stmt_generation (vinfo, stmt_info,
9251 : new_stmt, gsi);
9252 : }
9253 :
9254 30 : new_stmt = vect_build_one_scatter_store_call
9255 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9256 30 : dataref_ptr, vec_offsets[2 * j + 1],
9257 : vec_oprnd, final_mask);
9258 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9259 : new_stmt, gsi);
9260 30 : }
9261 46 : else if (known_eq (nunits * 2, offset_nunits))
9262 : {
9263 : /* We have a offset vector with double the number of
9264 : lanes. Select the low/high part accordingly. */
9265 46 : vec_offset = vec_offsets[j / 2];
9266 46 : if (j & 1)
9267 : {
9268 23 : int count = offset_nunits.to_constant ();
9269 23 : vec_perm_builder sel (count, count, 1);
9270 23 : sel.quick_grow (count);
9271 286 : for (int i = 0; i < count; ++i)
9272 240 : sel[i] = i | (count / 2);
9273 23 : vec_perm_indices indices (sel, 2, count);
9274 23 : tree perm_mask = vect_gen_perm_mask_checked
9275 23 : (TREE_TYPE (vec_offset), indices);
9276 23 : new_stmt = gimple_build_assign (NULL_TREE,
9277 : VEC_PERM_EXPR,
9278 : vec_offset,
9279 : vec_offset,
9280 : perm_mask);
9281 23 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
9282 23 : gimple_set_lhs (new_stmt, vec_offset);
9283 23 : vect_finish_stmt_generation (vinfo, stmt_info,
9284 : new_stmt, gsi);
9285 23 : }
9286 :
9287 46 : new_stmt = vect_build_one_scatter_store_call
9288 46 : (vinfo, stmt_info, slp_node, gsi,
9289 : ls.gs.decl, dataref_ptr, vec_offset,
9290 : vec_oprnd, final_mask);
9291 46 : vect_finish_stmt_generation (vinfo, stmt_info,
9292 : new_stmt, gsi);
9293 : }
9294 : else
9295 0 : gcc_unreachable ();
9296 : }
9297 : else
9298 : {
9299 : /* Emulated scatter. */
9300 1309 : gcc_assert (!final_mask);
9301 1309 : if (costing_p)
9302 : {
9303 931 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9304 : /* For emulated scatter N offset vector element extracts
9305 : (we assume the scalar scaling and ptr + offset add is
9306 : consumed by the load). */
9307 931 : inside_cost
9308 931 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9309 : slp_node, 0, vect_body);
9310 : /* N scalar stores plus extracting the elements. */
9311 931 : inside_cost
9312 931 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9313 : slp_node, 0, vect_body);
9314 931 : inside_cost
9315 931 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9316 : slp_node, 0, vect_body);
9317 931 : continue;
9318 931 : }
9319 :
9320 378 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9321 378 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
9322 378 : unsigned HOST_WIDE_INT const_offset_nunits
9323 378 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
9324 378 : vec<constructor_elt, va_gc> *ctor_elts;
9325 378 : vec_alloc (ctor_elts, const_nunits);
9326 378 : gimple_seq stmts = NULL;
9327 378 : tree elt_type = TREE_TYPE (vectype);
9328 378 : unsigned HOST_WIDE_INT elt_size
9329 378 : = tree_to_uhwi (TYPE_SIZE (elt_type));
9330 : /* We support offset vectors with more elements
9331 : than the data vector for now. */
9332 378 : unsigned HOST_WIDE_INT factor
9333 : = const_offset_nunits / const_nunits;
9334 378 : vec_offset = vec_offsets[j / factor];
9335 378 : unsigned elt_offset
9336 378 : = (j % factor) * const_nunits;
9337 378 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
9338 378 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9339 378 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
9340 1914 : for (unsigned k = 0; k < const_nunits; ++k)
9341 : {
9342 : /* Compute the offsetted pointer. */
9343 1158 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
9344 : bitsize_int (k + elt_offset));
9345 1158 : tree idx
9346 2316 : = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
9347 1158 : vec_offset, TYPE_SIZE (idx_type), boff);
9348 1158 : idx = gimple_convert (&stmts, sizetype, idx);
9349 1158 : idx = gimple_build (&stmts, MULT_EXPR, sizetype,
9350 : idx, scale);
9351 1158 : tree ptr
9352 1158 : = gimple_build (&stmts, PLUS_EXPR,
9353 1158 : TREE_TYPE (dataref_ptr),
9354 : dataref_ptr, idx);
9355 1158 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
9356 : /* Extract the element to be stored. */
9357 1158 : tree elt
9358 2316 : = gimple_build (&stmts, BIT_FIELD_REF,
9359 1158 : TREE_TYPE (vectype),
9360 1158 : vec_oprnd, TYPE_SIZE (elt_type),
9361 1158 : bitsize_int (k * elt_size));
9362 1158 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9363 1158 : stmts = NULL;
9364 1158 : tree ref
9365 1158 : = build2 (MEM_REF, ltype, ptr,
9366 : build_int_cst (ref_type, 0));
9367 1158 : new_stmt = gimple_build_assign (ref, elt);
9368 1158 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9369 : }
9370 :
9371 378 : slp_node->push_vec_def (new_stmt);
9372 : }
9373 : }
9374 :
9375 1383 : if (costing_p)
9376 : {
9377 909 : if (dump_enabled_p ())
9378 78 : dump_printf_loc (MSG_NOTE, vect_location,
9379 : "vect_model_store_cost: inside_cost = %d, "
9380 : "prologue_cost = %d .\n",
9381 : inside_cost, prologue_cost);
9382 909 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9383 909 : slp_node->data = new vect_load_store_data (std::move (ls));
9384 : }
9385 :
9386 1383 : return true;
9387 1383 : }
9388 :
9389 1355015 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
9390 : || memory_access_type == VMAT_CONTIGUOUS_DOWN
9391 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
9392 :
9393 1355015 : unsigned inside_cost = 0, prologue_cost = 0;
9394 : /* For costing some adjacent vector stores, we'd like to cost with
9395 : the total number of them once instead of cost each one by one. */
9396 1355015 : unsigned int n_adjacent_stores = 0;
9397 1355015 : auto_vec<tree> result_chain (group_size);
9398 1355015 : auto_vec<tree, 1> vec_oprnds;
9399 1355015 : gimple *new_stmt;
9400 1355015 : if (!costing_p)
9401 : {
9402 : /* Get vectorized arguments for SLP_NODE. */
9403 546073 : vect_get_slp_defs (op_node, &vec_oprnds);
9404 546073 : vec_oprnd = vec_oprnds[0];
9405 546073 : if (mask_node)
9406 : {
9407 470 : vect_get_slp_defs (mask_node, &vec_masks);
9408 470 : vec_mask = vec_masks[0];
9409 : }
9410 : }
9411 :
9412 : /* We should have caught mismatched types earlier. */
9413 546073 : gcc_assert (costing_p
9414 : || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
9415 1355015 : bool simd_lane_access_p
9416 1355015 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
9417 1355015 : if (!costing_p
9418 1355015 : && simd_lane_access_p
9419 4365 : && !loop_masks
9420 4365 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
9421 4365 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
9422 4365 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
9423 4365 : && integer_zerop (DR_INIT (first_dr_info->dr))
9424 1359380 : && alias_sets_conflict_p (get_alias_set (aggr_type),
9425 4365 : get_alias_set (TREE_TYPE (ref_type))))
9426 : {
9427 4357 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
9428 4357 : dataref_offset = build_int_cst (ref_type, 0);
9429 : }
9430 1350658 : else if (!costing_p)
9431 1083424 : dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
9432 : simd_lane_access_p ? loop : NULL,
9433 : offset, &dummy, gsi, NULL,
9434 : simd_lane_access_p, dr_increment);
9435 :
9436 1355015 : new_stmt = NULL;
9437 1355015 : gcc_assert (!grouped_store);
9438 3012704 : for (i = 0; i < vec_num; i++)
9439 : {
9440 1657689 : if (!costing_p)
9441 677103 : vec_oprnd = vec_oprnds[i];
9442 :
9443 1657689 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
9444 : {
9445 3391 : if (costing_p)
9446 2241 : inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
9447 : slp_node, 0, vect_body);
9448 : else
9449 : {
9450 1150 : tree perm_mask = perm_mask_for_reverse (vectype);
9451 1150 : tree new_temp = make_ssa_name (vectype);
9452 :
9453 : /* Generate the permute statement. */
9454 1150 : gimple *perm_stmt
9455 1150 : = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
9456 : vec_oprnd, perm_mask);
9457 1150 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9458 :
9459 1150 : perm_stmt = SSA_NAME_DEF_STMT (new_temp);
9460 1657689 : vec_oprnd = new_temp;
9461 : }
9462 : }
9463 :
9464 1657689 : if (costing_p)
9465 : {
9466 980586 : n_adjacent_stores++;
9467 980586 : continue;
9468 : }
9469 :
9470 677103 : tree final_mask = NULL_TREE;
9471 677103 : tree final_len = NULL_TREE;
9472 677103 : tree bias = NULL_TREE;
9473 677103 : if (loop_masks)
9474 83 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
9475 : vec_num, vectype, i);
9476 677103 : if (vec_mask)
9477 672 : vec_mask = vec_masks[i];
9478 672 : if (vec_mask)
9479 672 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
9480 : vec_mask, gsi);
9481 :
9482 677103 : if (i > 0)
9483 : /* Bump the vector pointer. */
9484 131030 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
9485 : dr_bump);
9486 :
9487 677103 : unsigned misalign;
9488 677103 : unsigned HOST_WIDE_INT align;
9489 677103 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
9490 677103 : if (alignment_support_scheme == dr_aligned)
9491 : misalign = 0;
9492 310711 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
9493 : {
9494 162852 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
9495 162852 : misalign = 0;
9496 : }
9497 : else
9498 147859 : misalign = misalignment;
9499 677103 : if (dataref_offset == NULL_TREE
9500 671737 : && TREE_CODE (dataref_ptr) == SSA_NAME)
9501 188041 : set_ssa_name_alignment (dataref_ptr, align, misalign);
9502 677103 : align = least_bit_hwi (misalign | align);
9503 :
9504 : /* Compute IFN when LOOP_LENS or final_mask valid. */
9505 677103 : machine_mode vmode = TYPE_MODE (vectype);
9506 677103 : machine_mode new_vmode = vmode;
9507 677103 : internal_fn partial_ifn = IFN_LAST;
9508 677103 : if (loop_lens)
9509 : {
9510 0 : opt_machine_mode new_ovmode
9511 0 : = get_len_load_store_mode (vmode, false, &partial_ifn);
9512 0 : new_vmode = new_ovmode.require ();
9513 0 : unsigned factor
9514 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
9515 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
9516 : vec_num, vectype, i, factor, true);
9517 : }
9518 677103 : else if (final_mask)
9519 : {
9520 688 : if (!can_vec_mask_load_store_p (vmode,
9521 688 : TYPE_MODE (TREE_TYPE (final_mask)),
9522 : false, &partial_ifn))
9523 0 : gcc_unreachable ();
9524 : }
9525 :
9526 677103 : if (partial_ifn == IFN_MASK_LEN_STORE)
9527 : {
9528 0 : if (!final_len)
9529 : {
9530 : /* Pass VF value to 'len' argument of
9531 : MASK_LEN_STORE if LOOP_LENS is invalid. */
9532 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9533 : }
9534 0 : if (!final_mask)
9535 : {
9536 : /* Pass all ones value to 'mask' argument of
9537 : MASK_LEN_STORE if final_mask is invalid. */
9538 0 : mask_vectype = truth_type_for (vectype);
9539 0 : final_mask = build_minus_one_cst (mask_vectype);
9540 : }
9541 : }
9542 677103 : if (final_len)
9543 : {
9544 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9545 0 : bias = build_int_cst (intQI_type_node, biasval);
9546 : }
9547 :
9548 : /* Arguments are ready. Create the new vector stmt. */
9549 0 : if (final_len)
9550 : {
9551 0 : gcall *call;
9552 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9553 : /* Need conversion if it's wrapped with VnQI. */
9554 0 : if (vmode != new_vmode)
9555 : {
9556 0 : tree new_vtype
9557 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
9558 : new_vmode);
9559 0 : tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
9560 0 : vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
9561 0 : gassign *new_stmt
9562 0 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
9563 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9564 0 : vec_oprnd = var;
9565 : }
9566 :
9567 0 : if (partial_ifn == IFN_MASK_LEN_STORE)
9568 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
9569 : dataref_ptr, ptr, final_mask,
9570 : final_len, bias, vec_oprnd);
9571 : else
9572 0 : call = gimple_build_call_internal (IFN_LEN_STORE, 5,
9573 : dataref_ptr, ptr, final_len,
9574 : bias, vec_oprnd);
9575 0 : gimple_call_set_nothrow (call, true);
9576 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9577 0 : new_stmt = call;
9578 : }
9579 677103 : else if (final_mask)
9580 : {
9581 688 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9582 688 : gcall *call
9583 688 : = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
9584 : ptr, final_mask, vec_oprnd);
9585 688 : gimple_call_set_nothrow (call, true);
9586 688 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9587 688 : new_stmt = call;
9588 : }
9589 : else
9590 : {
9591 676415 : data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
9592 : dataref_offset ? dataref_offset
9593 : : build_int_cst (ref_type, 0));
9594 676415 : if (alignment_support_scheme == dr_aligned
9595 676415 : && align >= TYPE_ALIGN_UNIT (vectype))
9596 : ;
9597 : else
9598 310184 : TREE_TYPE (data_ref)
9599 620368 : = build_aligned_type (TREE_TYPE (data_ref),
9600 : align * BITS_PER_UNIT);
9601 676415 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
9602 676415 : new_stmt = gimple_build_assign (data_ref, vec_oprnd);
9603 676415 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9604 : }
9605 : }
9606 :
9607 1355015 : if (costing_p)
9608 : {
9609 808942 : if (n_adjacent_stores > 0)
9610 808942 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
9611 : alignment_support_scheme, misalignment,
9612 : &inside_cost, cost_vec);
9613 :
9614 : /* When vectorizing a store into the function result assign
9615 : a penalty if the function returns in a multi-register location.
9616 : In this case we assume we'll end up with having to spill the
9617 : vector result and do piecewise loads as a conservative estimate. */
9618 808942 : tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
9619 808942 : if (base
9620 808942 : && (TREE_CODE (base) == RESULT_DECL
9621 758521 : || (DECL_P (base) && cfun_returns (base)))
9622 871186 : && !aggregate_value_p (base, cfun->decl))
9623 : {
9624 11237 : rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
9625 : /* ??? Handle PARALLEL in some way. */
9626 11237 : if (REG_P (reg))
9627 : {
9628 11028 : int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
9629 : /* Assume that a single reg-reg move is possible and cheap,
9630 : do not account for vector to gp register move cost. */
9631 11028 : if (nregs > 1)
9632 : {
9633 : /* Spill. */
9634 10151 : prologue_cost
9635 10151 : += record_stmt_cost (cost_vec, 1, vector_store,
9636 : slp_node, 0, vect_epilogue);
9637 : /* Loads. */
9638 10151 : prologue_cost
9639 10151 : += record_stmt_cost (cost_vec, nregs, scalar_load,
9640 : slp_node, 0, vect_epilogue);
9641 : }
9642 : }
9643 : }
9644 808942 : if (dump_enabled_p ())
9645 14630 : dump_printf_loc (MSG_NOTE, vect_location,
9646 : "vect_model_store_cost: inside_cost = %d, "
9647 : "prologue_cost = %d .\n",
9648 : inside_cost, prologue_cost);
9649 :
9650 808942 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9651 808942 : slp_node->data = new vect_load_store_data (std::move (ls));
9652 : }
9653 :
9654 1355015 : return true;
9655 1387924 : }
9656 :
9657 : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9658 : VECTOR_CST mask. No checks are made that the target platform supports the
9659 : mask, so callers may wish to test can_vec_perm_const_p separately, or use
9660 : vect_gen_perm_mask_checked. */
9661 :
9662 : tree
9663 62072 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
9664 : {
9665 62072 : tree mask_type;
9666 :
9667 62072 : poly_uint64 nunits = sel.length ();
9668 62072 : gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
9669 :
9670 62072 : mask_type = build_vector_type (ssizetype, nunits);
9671 62072 : return vec_perm_indices_to_tree (mask_type, sel);
9672 : }
9673 :
9674 : /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9675 : i.e. that the target supports the pattern _for arbitrary input vectors_. */
9676 :
9677 : tree
9678 59387 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
9679 : {
9680 59387 : machine_mode vmode = TYPE_MODE (vectype);
9681 59387 : gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
9682 59387 : return vect_gen_perm_mask_any (vectype, sel);
9683 : }
9684 :
9685 : /* Given a vector variable X and Y, that was generated for the scalar
9686 : STMT_INFO, generate instructions to permute the vector elements of X and Y
9687 : using permutation mask MASK_VEC, insert them at *GSI and return the
9688 : permuted vector variable. */
9689 :
9690 : static tree
9691 1488 : permute_vec_elements (vec_info *vinfo,
9692 : tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
9693 : gimple_stmt_iterator *gsi)
9694 : {
9695 1488 : tree vectype = TREE_TYPE (x);
9696 1488 : tree perm_dest, data_ref;
9697 1488 : gimple *perm_stmt;
9698 :
9699 1488 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
9700 1488 : if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
9701 1488 : perm_dest = vect_create_destination_var (scalar_dest, vectype);
9702 : else
9703 0 : perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
9704 1488 : data_ref = make_ssa_name (perm_dest);
9705 :
9706 : /* Generate the permute statement. */
9707 1488 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
9708 1488 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9709 :
9710 1488 : return data_ref;
9711 : }
9712 :
9713 : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9714 : inserting them on the loops preheader edge. Returns true if we
9715 : were successful in doing so (and thus STMT_INFO can be moved then),
9716 : otherwise returns false. HOIST_P indicates if we want to hoist the
9717 : definitions of all SSA uses, it would be false when we are costing. */
9718 :
9719 : static bool
9720 4032 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
9721 : {
9722 4032 : ssa_op_iter i;
9723 4032 : use_operand_p use_p;
9724 4032 : auto_vec<use_operand_p, 8> to_hoist;
9725 :
9726 7640 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
9727 : {
9728 3638 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9729 3638 : if (!gimple_nop_p (def_stmt)
9730 3638 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
9731 : {
9732 : /* Make sure we don't need to recurse. While we could do
9733 : so in simple cases when there are more complex use webs
9734 : we don't have an easy way to preserve stmt order to fulfil
9735 : dependencies within them. */
9736 87 : tree op2;
9737 87 : ssa_op_iter i2;
9738 87 : if (gimple_code (def_stmt) == GIMPLE_PHI
9739 87 : || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
9740 : == NULL_DEF_OPERAND_P))
9741 30 : return false;
9742 176 : FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
9743 : {
9744 119 : gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
9745 119 : if (!gimple_nop_p (def_stmt2)
9746 119 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
9747 : return false;
9748 : }
9749 57 : to_hoist.safe_push (use_p);
9750 : }
9751 : }
9752 :
9753 8034 : if (to_hoist.is_empty ())
9754 : return true;
9755 :
9756 33 : if (!hoist_p)
9757 : return true;
9758 :
9759 : /* Instead of moving defs we copy them so we can zero their UID to not
9760 : confuse dominance queries in the preheader. */
9761 3 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
9762 12 : for (use_operand_p use_p : to_hoist)
9763 : {
9764 3 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9765 3 : gimple *copy = gimple_copy (def_stmt);
9766 3 : assign_discriminators_to_stmt
9767 3 : (copy, 0, allocate_copyid_base (gimple_location (copy), 1));
9768 3 : gimple_set_uid (copy, 0);
9769 3 : def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
9770 3 : tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
9771 3 : update_stmt (copy);
9772 3 : def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
9773 3 : SET_DEF (def_p, new_def);
9774 3 : SET_USE (use_p, new_def);
9775 3 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
9776 : }
9777 :
9778 : return true;
9779 4032 : }
9780 :
9781 : /* vectorizable_load.
9782 :
9783 : Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9784 : that can be vectorized.
9785 : If COST_VEC is passed, calculate costs but don't change anything,
9786 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
9787 : it, and insert it at GSI.
9788 : Return true if STMT_INFO is vectorizable in this way. */
9789 :
9790 : static bool
9791 2240149 : vectorizable_load (vec_info *vinfo,
9792 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
9793 : slp_tree slp_node,
9794 : stmt_vector_for_cost *cost_vec)
9795 : {
9796 2240149 : tree scalar_dest;
9797 2240149 : tree vec_dest = NULL;
9798 2240149 : tree data_ref = NULL;
9799 2240149 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
9800 2240149 : class loop *loop = NULL;
9801 2240149 : class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
9802 2240149 : bool nested_in_vect_loop = false;
9803 2240149 : tree elem_type;
9804 : /* Avoid false positive uninitialized warning, see PR110652. */
9805 2240149 : tree new_temp = NULL_TREE;
9806 2240149 : machine_mode mode;
9807 2240149 : tree dummy;
9808 2240149 : tree dataref_ptr = NULL_TREE;
9809 2240149 : tree dataref_offset = NULL_TREE;
9810 2240149 : int i, j;
9811 2240149 : unsigned int group_size;
9812 2240149 : poly_uint64 group_gap_adj;
9813 2240149 : tree msq = NULL_TREE, lsq;
9814 2240149 : tree realignment_token = NULL_TREE;
9815 2240149 : gphi *phi = NULL;
9816 2240149 : bool grouped_load = false;
9817 2240149 : stmt_vec_info first_stmt_info;
9818 2240149 : stmt_vec_info first_stmt_info_for_drptr = NULL;
9819 2240149 : bool compute_in_loop = false;
9820 2240149 : class loop *at_loop;
9821 2240149 : int vec_num;
9822 2240149 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
9823 2240149 : poly_uint64 vf;
9824 2240149 : tree aggr_type;
9825 2240149 : tree ref_type;
9826 2240149 : enum vect_def_type mask_dt = vect_unknown_def_type;
9827 2240149 : enum vect_def_type els_dt = vect_unknown_def_type;
9828 :
9829 2240149 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
9830 : return false;
9831 :
9832 2240149 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
9833 251571 : && cost_vec)
9834 : return false;
9835 :
9836 1988578 : if (!STMT_VINFO_DATA_REF (stmt_info))
9837 : return false;
9838 :
9839 1579798 : tree mask_vectype = NULL_TREE;
9840 1579798 : tree els = NULL_TREE; tree els_vectype = NULL_TREE;
9841 :
9842 1579798 : int mask_index = -1;
9843 1579798 : int els_index = -1;
9844 1579798 : slp_tree mask_node = NULL;
9845 1579798 : slp_tree els_op = NULL;
9846 1579798 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
9847 : {
9848 1575659 : scalar_dest = gimple_assign_lhs (assign);
9849 1575659 : if (TREE_CODE (scalar_dest) != SSA_NAME)
9850 : return false;
9851 :
9852 739558 : tree_code code = gimple_assign_rhs_code (assign);
9853 739558 : if (code != ARRAY_REF
9854 739558 : && code != BIT_FIELD_REF
9855 739558 : && code != INDIRECT_REF
9856 507998 : && code != COMPONENT_REF
9857 507998 : && code != IMAGPART_EXPR
9858 367072 : && code != REALPART_EXPR
9859 367072 : && code != MEM_REF
9860 285 : && TREE_CODE_CLASS (code) != tcc_declaration)
9861 : return false;
9862 : }
9863 : else
9864 : {
9865 4139 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
9866 4139 : if (!call || !gimple_call_internal_p (call))
9867 : return false;
9868 :
9869 4139 : internal_fn ifn = gimple_call_internal_fn (call);
9870 4139 : if (!internal_load_fn_p (ifn))
9871 : return false;
9872 :
9873 2796 : scalar_dest = gimple_call_lhs (call);
9874 2796 : if (!scalar_dest)
9875 : return false;
9876 :
9877 2796 : mask_index = internal_fn_mask_index (ifn);
9878 2796 : if (mask_index >= 0)
9879 2796 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
9880 2796 : if (mask_index >= 0
9881 2796 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
9882 : &mask_node, &mask_dt, &mask_vectype))
9883 : return false;
9884 :
9885 2796 : els_index = internal_fn_else_index (ifn);
9886 2796 : if (els_index >= 0)
9887 2796 : els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
9888 2796 : if (els_index >= 0
9889 2796 : && !vect_is_simple_use (vinfo, slp_node, els_index,
9890 : &els, &els_op, &els_dt, &els_vectype))
9891 : return false;
9892 : }
9893 :
9894 742287 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9895 742287 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9896 :
9897 742287 : if (loop_vinfo)
9898 : {
9899 523676 : loop = LOOP_VINFO_LOOP (loop_vinfo);
9900 523676 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
9901 523676 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9902 : }
9903 : else
9904 : vf = 1;
9905 :
9906 742287 : vec_num = vect_get_num_copies (vinfo, slp_node);
9907 :
9908 : /* FORNOW. This restriction should be relaxed. */
9909 742287 : if (nested_in_vect_loop && vec_num > 1)
9910 : {
9911 319 : if (dump_enabled_p ())
9912 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9913 : "multiple types in nested loop.\n");
9914 : return false;
9915 : }
9916 :
9917 741968 : elem_type = TREE_TYPE (vectype);
9918 741968 : mode = TYPE_MODE (vectype);
9919 :
9920 : /* FORNOW. In some cases can vectorize even if data-type not supported
9921 : (e.g. - data copies). */
9922 741968 : if (!can_implement_p (mov_optab, mode))
9923 : {
9924 0 : if (dump_enabled_p ())
9925 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9926 : "Aligned load, but unsupported type.\n");
9927 : return false;
9928 : }
9929 :
9930 : /* Check if the load is a part of an interleaving chain. */
9931 741968 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
9932 : {
9933 310874 : grouped_load = true;
9934 : /* FORNOW */
9935 310874 : gcc_assert (!nested_in_vect_loop);
9936 310874 : gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
9937 :
9938 310874 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
9939 310874 : group_size = DR_GROUP_SIZE (first_stmt_info);
9940 :
9941 : /* Invalidate assumptions made by dependence analysis when vectorization
9942 : on the unrolled body effectively re-orders stmts. */
9943 310874 : if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
9944 310874 : && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
9945 : STMT_VINFO_MIN_NEG_DIST (stmt_info)))
9946 : {
9947 0 : if (dump_enabled_p ())
9948 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9949 : "cannot perform implicit CSE when performing "
9950 : "group loads with negative dependence distance\n");
9951 : return false;
9952 : }
9953 : }
9954 : else
9955 : group_size = 1;
9956 :
9957 741968 : vect_load_store_data _ls_data{};
9958 741968 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
9959 741968 : if (cost_vec
9960 741968 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
9961 : VLS_LOAD, &ls))
9962 : return false;
9963 : /* Temporary aliases to analysis data, should not be modified through
9964 : these. */
9965 623767 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
9966 623767 : const dr_alignment_support alignment_support_scheme
9967 : = ls.alignment_support_scheme;
9968 623767 : const int misalignment = ls.misalignment;
9969 623767 : const poly_int64 poffset = ls.poffset;
9970 623767 : const vec<int> &elsvals = ls.elsvals;
9971 :
9972 623767 : int maskload_elsval = 0;
9973 623767 : bool need_zeroing = false;
9974 :
9975 : /* We might need to explicitly zero inactive elements if there are
9976 : padding bits in the type that might leak otherwise.
9977 : Refer to PR115336. */
9978 623767 : tree scalar_type = TREE_TYPE (scalar_dest);
9979 623767 : bool type_mode_padding_p
9980 1247534 : = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
9981 :
9982 623767 : if (TREE_CODE (scalar_type) == BITINT_TYPE
9983 623767 : && !type_has_mode_precision_p (scalar_type))
9984 : {
9985 140 : if (dump_enabled_p ())
9986 18 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9987 : "_BitInt loads are not fully supported.\n");
9988 : return false;
9989 : }
9990 :
9991 623627 : if (slp_node->ldst_lanes
9992 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
9993 : {
9994 0 : if (dump_enabled_p ())
9995 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9996 : "discovered load-lane but cannot use it.\n");
9997 : return false;
9998 : }
9999 :
10000 623627 : if (mask_node)
10001 : {
10002 2670 : if (memory_access_type == VMAT_CONTIGUOUS)
10003 : {
10004 2105 : machine_mode vec_mode = TYPE_MODE (vectype);
10005 730 : if (!VECTOR_MODE_P (vec_mode)
10006 4210 : || !can_vec_mask_load_store_p (vec_mode,
10007 2105 : TYPE_MODE (mask_vectype),
10008 : true, NULL, &ls.elsvals))
10009 : return false;
10010 : }
10011 565 : else if (memory_access_type == VMAT_ELEMENTWISE
10012 565 : || memory_access_type == VMAT_STRIDED_SLP)
10013 : {
10014 0 : if (dump_enabled_p ())
10015 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10016 : "unsupported masked strided access.\n");
10017 : return false;
10018 : }
10019 565 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
10020 565 : && !mat_gather_scatter_p (memory_access_type))
10021 : {
10022 32 : if (dump_enabled_p ())
10023 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10024 : "unsupported access type for masked load.\n");
10025 : return false;
10026 : }
10027 533 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
10028 : {
10029 212 : if (dump_enabled_p ())
10030 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10031 : "unsupported masked emulated gather.\n");
10032 : return false;
10033 : }
10034 : }
10035 :
10036 623028 : bool costing_p = cost_vec;
10037 :
10038 623028 : if (costing_p) /* transformation not required. */
10039 : {
10040 465362 : if (loop_vinfo
10041 329757 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
10042 224730 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
10043 : VLS_LOAD, group_size, &ls,
10044 : mask_node, &ls.elsvals);
10045 :
10046 : /* If the type needs padding we must zero inactive elements.
10047 : Check if we can do that with a VEC_COND_EXPR and store the
10048 : elsval we choose in MASKLOAD_ELSVAL. */
10049 465362 : if (ls.elsvals.length ()
10050 62129 : && type_mode_padding_p
10051 7 : && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
10052 62129 : && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
10053 : {
10054 0 : if (dump_enabled_p ())
10055 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10056 : "cannot zero inactive elements.\n");
10057 : return false;
10058 : }
10059 :
10060 465362 : if (mask_node
10061 465362 : && !vect_maybe_update_slp_op_vectype (mask_node,
10062 : mask_vectype))
10063 : {
10064 0 : if (dump_enabled_p ())
10065 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10066 : "incompatible vector types for invariants\n");
10067 : return false;
10068 : }
10069 :
10070 465362 : if (dump_enabled_p ()
10071 27327 : && memory_access_type != VMAT_ELEMENTWISE
10072 27216 : && !mat_gather_scatter_p (memory_access_type)
10073 26901 : && memory_access_type != VMAT_STRIDED_SLP
10074 26901 : && memory_access_type != VMAT_INVARIANT
10075 491292 : && alignment_support_scheme != dr_aligned)
10076 10774 : dump_printf_loc (MSG_NOTE, vect_location,
10077 : "Vectorizing an unaligned access.\n");
10078 :
10079 465362 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10080 0 : vinfo->any_known_not_updated_vssa = true;
10081 : }
10082 :
10083 : /* For now just use the first available else value.
10084 : get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
10085 : select it here if it is supported. */
10086 623028 : if (elsvals.length ())
10087 85783 : maskload_elsval = *elsvals.begin ();
10088 :
10089 623028 : if (dump_enabled_p () && !costing_p)
10090 17214 : dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
10091 :
10092 : /* Transform. */
10093 :
10094 623028 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
10095 623028 : if (!costing_p)
10096 157666 : ensure_base_align (dr_info);
10097 :
10098 623028 : if (memory_access_type == VMAT_INVARIANT)
10099 : {
10100 5942 : gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
10101 : /* If we have versioned for aliasing or the loop doesn't
10102 : have any data dependencies that would preclude this,
10103 : then we are sure this is a loop invariant load and
10104 : thus we can insert it on the preheader edge.
10105 : TODO: hoist_defs_of_uses should ideally be computed
10106 : once at analysis time, remembered and used in the
10107 : transform time. */
10108 11884 : bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
10109 5942 : && !nested_in_vect_loop);
10110 :
10111 5942 : bool uniform_p = true;
10112 24592 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10113 : {
10114 : /* It is unsafe to hoist a conditional load over the conditions that
10115 : make it valid. When early break this means that any invariant load
10116 : can't be hoisted unless it's in the loop header or if we know
10117 : something else has verified the load is valid to do. Alignment
10118 : peeling would do this since getting through the prologue means the
10119 : load was done at least once and so the vector main body is free to
10120 : hoist it. However today GCC will hoist the load above the PFA
10121 : loop. As such that makes it still invalid and so we can't allow it
10122 : today. */
10123 6766 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10124 2912 : && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
10125 9642 : && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
10126 2876 : != loop->header)
10127 : {
10128 2778 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
10129 2778 : && dump_enabled_p ())
10130 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10131 : "not hoisting invariant load due to early break"
10132 : "constraints\n");
10133 2772 : else if (dump_enabled_p ())
10134 26 : dump_printf_loc (MSG_NOTE, vect_location,
10135 : "not hoisting invariant load due to early break"
10136 : "constraints\n");
10137 : hoist_p = false;
10138 : }
10139 :
10140 4020 : hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
10141 6766 : if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
10142 321 : uniform_p = false;
10143 : }
10144 5942 : if (costing_p)
10145 : {
10146 5112 : if (!uniform_p && (!hoist_p || !vf.is_constant ()))
10147 : {
10148 0 : if (dump_enabled_p ())
10149 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10150 : "not vectorizing non-uniform invariant "
10151 : "load\n");
10152 : return false;
10153 : }
10154 3292 : enum vect_cost_model_location cost_loc
10155 5112 : = hoist_p ? vect_prologue : vect_body;
10156 5236 : unsigned int cost = record_stmt_cost (cost_vec,
10157 : uniform_p
10158 124 : ? 1 : SLP_TREE_LANES (slp_node),
10159 : scalar_load,
10160 : slp_node, 0, cost_loc);
10161 5112 : cost += record_stmt_cost (cost_vec, 1,
10162 : uniform_p ? scalar_to_vec : vec_construct,
10163 : slp_node, 0, cost_loc);
10164 5112 : unsigned int prologue_cost = hoist_p ? cost : 0;
10165 3292 : unsigned int inside_cost = hoist_p ? 0 : cost;
10166 5112 : if (dump_enabled_p ())
10167 560 : dump_printf_loc (MSG_NOTE, vect_location,
10168 : "vect_model_load_cost: inside_cost = %d, "
10169 : "prologue_cost = %d .\n",
10170 : inside_cost, prologue_cost);
10171 5112 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10172 5112 : slp_node->data = new vect_load_store_data (std::move (ls));
10173 5112 : return true;
10174 : }
10175 830 : if (hoist_p)
10176 : {
10177 : /* ??? For non-uniform lanes there could be still duplicates.
10178 : We're leaving those to post-vectorizer CSE for the moment. */
10179 629 : auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
10180 2054 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10181 : {
10182 733 : gassign *stmt = as_a <gassign *> (sinfo->stmt);
10183 733 : if (dump_enabled_p ())
10184 379 : dump_printf_loc (MSG_NOTE, vect_location,
10185 : "hoisting out of the vectorized loop: %G",
10186 : (gimple *) stmt);
10187 733 : scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
10188 733 : tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
10189 733 : edge pe = loop_preheader_edge (loop);
10190 733 : gphi *vphi = get_virtual_phi (loop->header);
10191 733 : tree vuse;
10192 733 : if (vphi)
10193 726 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
10194 : else
10195 7 : vuse = gimple_vuse (gsi_stmt (*gsi));
10196 733 : gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
10197 733 : gimple_set_vuse (new_stmt, vuse);
10198 733 : gsi_insert_on_edge_immediate (pe, new_stmt);
10199 733 : hoist_defs_of_uses (new_stmt, loop, true);
10200 733 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
10201 733 : TREE_TYPE (scalar_dest)))
10202 : {
10203 10 : tree tem = make_ssa_name (TREE_TYPE (vectype));
10204 10 : new_stmt = gimple_build_assign (tem,
10205 : NOP_EXPR, scalar_dest);
10206 10 : gsi_insert_on_edge_immediate (pe, new_stmt);
10207 10 : scalar_dest = tem;
10208 : }
10209 733 : scalar_defs.quick_push (scalar_dest);
10210 733 : if (uniform_p)
10211 : break;
10212 : }
10213 629 : if (!uniform_p)
10214 : {
10215 63 : unsigned const_nunits
10216 63 : = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10217 144 : for (j = 0; j < (int) vec_num; ++j)
10218 : {
10219 81 : vec<constructor_elt, va_gc> *v = NULL;
10220 81 : vec_safe_reserve (v, const_nunits, true);
10221 522 : for (unsigned i = 0; i < const_nunits; ++i)
10222 : {
10223 360 : unsigned def_idx
10224 360 : = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
10225 360 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
10226 : scalar_defs[def_idx]);
10227 : }
10228 81 : scalar_dest = build_constructor (vectype, v);
10229 81 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10230 : vectype, NULL);
10231 81 : slp_node->push_vec_def (new_temp);
10232 : }
10233 63 : return true;
10234 : }
10235 566 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10236 : vectype, NULL);
10237 629 : }
10238 : else
10239 : {
10240 201 : gcc_assert (uniform_p);
10241 201 : gimple_stmt_iterator gsi2 = *gsi;
10242 201 : gsi_next (&gsi2);
10243 201 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10244 : vectype, &gsi2);
10245 : }
10246 1609 : for (j = 0; j < (int) vec_num; ++j)
10247 842 : slp_node->push_vec_def (new_temp);
10248 : return true;
10249 : }
10250 :
10251 617086 : if (memory_access_type == VMAT_ELEMENTWISE
10252 617086 : || memory_access_type == VMAT_STRIDED_SLP)
10253 : {
10254 24458 : gimple_stmt_iterator incr_gsi;
10255 24458 : bool insert_after;
10256 24458 : tree offvar = NULL_TREE;
10257 24458 : tree ivstep;
10258 24458 : tree running_off;
10259 24458 : vec<constructor_elt, va_gc> *v = NULL;
10260 24458 : tree stride_base, stride_step = NULL_TREE, alias_off;
10261 : /* Checked by get_load_store_type. */
10262 24458 : unsigned int const_nunits = nunits.to_constant ();
10263 24458 : unsigned HOST_WIDE_INT cst_offset = 0;
10264 24458 : tree dr_offset;
10265 24458 : unsigned int inside_cost = 0;
10266 :
10267 24458 : gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
10268 24458 : gcc_assert (!nested_in_vect_loop);
10269 :
10270 24458 : if (grouped_load)
10271 : {
10272 : /* If we elided a consecutive load permutation, don't
10273 : use the original first statement (which could be elided)
10274 : but the one the load permutation starts with.
10275 : This ensures the stride_base below is correct. */
10276 11150 : if (!ls.subchain_p)
10277 11068 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10278 : else
10279 82 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10280 11150 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10281 11150 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10282 : }
10283 : else
10284 : {
10285 13308 : first_stmt_info = stmt_info;
10286 13308 : first_dr_info = dr_info;
10287 13308 : ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
10288 : }
10289 :
10290 24458 : if (grouped_load)
10291 : {
10292 11150 : if (memory_access_type == VMAT_STRIDED_SLP)
10293 : {
10294 : /* If we elided a consecutive load permutation, adjust
10295 : the group size here. */
10296 4291 : if (!ls.subchain_p)
10297 4209 : group_size = DR_GROUP_SIZE (first_stmt_info);
10298 : else
10299 82 : group_size = SLP_TREE_LANES (slp_node);
10300 : }
10301 : else /* VMAT_ELEMENTWISE */
10302 6859 : group_size = SLP_TREE_LANES (slp_node);
10303 : }
10304 : else
10305 : group_size = 1;
10306 :
10307 24458 : if (!costing_p)
10308 : {
10309 3563 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
10310 3563 : stride_base = fold_build_pointer_plus (
10311 : DR_BASE_ADDRESS (first_dr_info->dr),
10312 : size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
10313 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
10314 3563 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
10315 :
10316 : /* For a load with loop-invariant (but other than power-of-2)
10317 : stride (i.e. not a grouped access) like so:
10318 :
10319 : for (i = 0; i < n; i += stride)
10320 : ... = array[i];
10321 :
10322 : we generate a new induction variable and new accesses to
10323 : form a new vector (or vectors, depending on ncopies):
10324 :
10325 : for (j = 0; ; j += VF*stride)
10326 : tmp1 = array[j];
10327 : tmp2 = array[j + stride];
10328 : ...
10329 : vectemp = {tmp1, tmp2, ...}
10330 : */
10331 :
10332 3563 : tree increment = fold_convert (TREE_TYPE (stride_step),
10333 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
10334 3563 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
10335 : stride_step, increment);
10336 :
10337 3563 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
10338 :
10339 3563 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
10340 3563 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10341 3563 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
10342 : else
10343 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
10344 : true, NULL_TREE, true,
10345 : GSI_SAME_STMT);
10346 3563 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
10347 : insert_after, &offvar, NULL, true);
10348 :
10349 3563 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
10350 : }
10351 :
10352 24458 : running_off = offvar;
10353 24458 : alias_off = build_int_cst (ref_type, 0);
10354 24458 : int nloads = const_nunits;
10355 24458 : int lnel = 1;
10356 24458 : tree ltype = TREE_TYPE (vectype);
10357 24458 : tree lvectype = vectype;
10358 24458 : auto_vec<tree> dr_chain;
10359 : /* ??? Modify local copies of alignment_support_scheme and
10360 : misalignment, but this part of analysis should be done
10361 : earlier and remembered, likewise the chosen load mode. */
10362 24458 : const dr_alignment_support tem = alignment_support_scheme;
10363 24458 : dr_alignment_support alignment_support_scheme = tem;
10364 24458 : const int tem2 = misalignment;
10365 24458 : int misalignment = tem2;
10366 24458 : if (memory_access_type == VMAT_STRIDED_SLP)
10367 : {
10368 17599 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
10369 : /* Use the target vector type if the group size is a multiple
10370 : of it. */
10371 17599 : if (n == const_nunits)
10372 : {
10373 2266 : int mis_align = dr_misalignment (first_dr_info, vectype);
10374 : /* With VF > 1 we advance the DR by step, if that is constant
10375 : and only aligned when performed VF times, DR alignment
10376 : analysis can analyze this as aligned since it assumes
10377 : contiguous accesses. But that is not how we code generate
10378 : here, so adjust for this. */
10379 2266 : if (maybe_gt (vf, 1u)
10380 3634 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10381 3418 : DR_TARGET_ALIGNMENT (first_dr_info)))
10382 216 : mis_align = -1;
10383 2266 : dr_alignment_support dr_align
10384 2266 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
10385 : mis_align);
10386 2266 : if (dr_align == dr_aligned
10387 2266 : || dr_align == dr_unaligned_supported)
10388 : {
10389 17599 : nloads = 1;
10390 17599 : lnel = const_nunits;
10391 17599 : ltype = vectype;
10392 17599 : alignment_support_scheme = dr_align;
10393 17599 : misalignment = mis_align;
10394 : }
10395 : }
10396 : /* Else use the biggest vector we can load the group without
10397 : accessing excess elements. */
10398 15333 : else if (n > 1)
10399 : {
10400 2014 : tree ptype;
10401 2014 : tree vtype
10402 2014 : = vector_vector_composition_type (vectype, const_nunits / n,
10403 : &ptype);
10404 2014 : if (vtype != NULL_TREE)
10405 : {
10406 1976 : dr_alignment_support dr_align;
10407 1976 : int mis_align = 0;
10408 1976 : if (VECTOR_TYPE_P (ptype))
10409 : {
10410 1056 : mis_align = dr_misalignment (first_dr_info, ptype);
10411 1056 : if (maybe_gt (vf, 1u)
10412 2082 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10413 1062 : DR_TARGET_ALIGNMENT (first_dr_info)))
10414 1020 : mis_align = -1;
10415 1056 : dr_align
10416 1056 : = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
10417 : mis_align);
10418 : }
10419 : else
10420 : dr_align = dr_unaligned_supported;
10421 1976 : if (dr_align == dr_aligned
10422 1976 : || dr_align == dr_unaligned_supported)
10423 : {
10424 1976 : nloads = const_nunits / n;
10425 1976 : lnel = n;
10426 1976 : lvectype = vtype;
10427 1976 : ltype = ptype;
10428 1976 : alignment_support_scheme = dr_align;
10429 1976 : misalignment = mis_align;
10430 : }
10431 : }
10432 : }
10433 17599 : unsigned align;
10434 17599 : if (alignment_support_scheme == dr_aligned)
10435 14 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
10436 : else
10437 17585 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
10438 : /* Alignment is at most the access size if we do multiple loads. */
10439 17599 : if (nloads > 1)
10440 15333 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
10441 17599 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
10442 : }
10443 :
10444 24458 : if (costing_p)
10445 : {
10446 : /* Record the composition type for target access during costing. */
10447 20895 : ls.ls_type = lvectype;
10448 20895 : ls.ls_eltype = ltype;
10449 : }
10450 : else
10451 3563 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
10452 :
10453 : /* For SLP permutation support we need to load the whole group,
10454 : not only the number of vector stmts the permutation result
10455 : fits in. */
10456 24458 : int ncopies;
10457 24458 : if (ls.slp_perm)
10458 : {
10459 2871 : gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
10460 : /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10461 : variable VF. */
10462 2871 : unsigned int const_vf = vf.to_constant ();
10463 2871 : ncopies = CEIL (group_size * const_vf, const_nunits);
10464 2871 : dr_chain.create (ncopies);
10465 : }
10466 : else
10467 : ncopies = vec_num;
10468 :
10469 24458 : unsigned int group_el = 0;
10470 24458 : unsigned HOST_WIDE_INT
10471 24458 : elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
10472 24458 : unsigned int n_groups = 0;
10473 : /* For costing some adjacent vector loads, we'd like to cost with
10474 : the total number of them once instead of cost each one by one. */
10475 24458 : unsigned int n_adjacent_loads = 0;
10476 58172 : for (j = 0; j < ncopies; j++)
10477 : {
10478 33714 : if (nloads > 1 && !costing_p)
10479 3275 : vec_alloc (v, nloads);
10480 33714 : gimple *new_stmt = NULL;
10481 141980 : for (i = 0; i < nloads; i++)
10482 : {
10483 108266 : if (costing_p)
10484 : {
10485 : /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10486 : avoid ICE, see PR110776. */
10487 97850 : if (VECTOR_TYPE_P (ltype)
10488 5918 : && memory_access_type != VMAT_ELEMENTWISE)
10489 5918 : n_adjacent_loads++;
10490 : else
10491 91932 : inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
10492 : slp_node, 0, vect_body);
10493 97850 : continue;
10494 : }
10495 10416 : unsigned int load_el = group_el;
10496 : /* For elementwise accesses apply a load permutation directly. */
10497 10416 : if (memory_access_type == VMAT_ELEMENTWISE
10498 10416 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10499 2266 : load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
10500 10416 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
10501 10416 : load_el * elsz + cst_offset);
10502 10416 : tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
10503 10416 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
10504 10416 : new_temp = make_ssa_name (ltype);
10505 10416 : new_stmt = gimple_build_assign (new_temp, data_ref);
10506 10416 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
10507 10416 : if (nloads > 1)
10508 8800 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
10509 :
10510 10416 : group_el += lnel;
10511 10416 : if (group_el == group_size)
10512 : {
10513 10047 : n_groups++;
10514 : /* When doing SLP make sure to not load elements from
10515 : the next vector iteration, those will not be accessed
10516 : so just use the last element again. See PR107451. */
10517 10047 : if (known_lt (n_groups, vf))
10518 : {
10519 6465 : tree newoff = copy_ssa_name (running_off);
10520 6465 : gimple *incr
10521 6465 : = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
10522 : running_off, stride_step);
10523 6465 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
10524 6465 : running_off = newoff;
10525 : }
10526 : group_el = 0;
10527 : }
10528 : }
10529 :
10530 33714 : if (nloads > 1)
10531 : {
10532 24956 : if (costing_p)
10533 21681 : inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
10534 : slp_node, 0, vect_body);
10535 : else
10536 : {
10537 3275 : tree vec_inv = build_constructor (lvectype, v);
10538 3275 : new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
10539 : lvectype, gsi);
10540 3275 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
10541 3275 : if (lvectype != vectype)
10542 : {
10543 398 : new_stmt
10544 398 : = gimple_build_assign (make_ssa_name (vectype),
10545 : VIEW_CONVERT_EXPR,
10546 : build1 (VIEW_CONVERT_EXPR,
10547 : vectype, new_temp));
10548 398 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10549 : gsi);
10550 : }
10551 : }
10552 : }
10553 8758 : else if (!costing_p && ltype != vectype)
10554 : {
10555 1599 : new_stmt = gimple_build_assign (make_ssa_name (vectype),
10556 : VIEW_CONVERT_EXPR,
10557 : build1 (VIEW_CONVERT_EXPR,
10558 : vectype, new_temp));
10559 1599 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10560 : gsi);
10561 : }
10562 :
10563 33714 : if (!costing_p)
10564 : {
10565 4891 : if (ls.slp_perm)
10566 1682 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
10567 : else
10568 3209 : slp_node->push_vec_def (new_stmt);
10569 : }
10570 : }
10571 24458 : if (ls.slp_perm)
10572 : {
10573 2871 : if (costing_p)
10574 : {
10575 2078 : gcc_assert (ls.n_perms != -1U);
10576 2078 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
10577 : slp_node, 0, vect_body);
10578 : }
10579 : else
10580 : {
10581 793 : unsigned n_perms2;
10582 793 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
10583 : false, &n_perms2);
10584 793 : gcc_assert (ls.n_perms == n_perms2);
10585 : }
10586 : }
10587 :
10588 24458 : if (costing_p)
10589 : {
10590 20895 : if (n_adjacent_loads > 0)
10591 2192 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10592 : alignment_support_scheme, misalignment, false,
10593 : &inside_cost, nullptr, cost_vec, cost_vec,
10594 : true);
10595 20895 : if (dump_enabled_p ())
10596 522 : dump_printf_loc (MSG_NOTE, vect_location,
10597 : "vect_model_load_cost: inside_cost = %u, "
10598 : "prologue_cost = 0 .\n",
10599 : inside_cost);
10600 20895 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10601 20895 : slp_node->data = new vect_load_store_data (std::move (ls));
10602 : }
10603 :
10604 24458 : return true;
10605 24458 : }
10606 :
10607 592628 : if (mat_gather_scatter_p (memory_access_type)
10608 592628 : && !ls.ls_type)
10609 : grouped_load = false;
10610 :
10611 589638 : if (grouped_load
10612 592628 : || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10613 : {
10614 261670 : if (grouped_load)
10615 : {
10616 261197 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10617 261197 : group_size = DR_GROUP_SIZE (first_stmt_info);
10618 : }
10619 : else
10620 : {
10621 : first_stmt_info = stmt_info;
10622 : group_size = 1;
10623 : }
10624 : /* For SLP vectorization we directly vectorize a subchain
10625 : without permutation. */
10626 261670 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10627 204194 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10628 : /* For BB vectorization always use the first stmt to base
10629 : the data ref pointer on. */
10630 261670 : if (bb_vinfo)
10631 210286 : first_stmt_info_for_drptr
10632 210286 : = vect_find_first_scalar_stmt_in_slp (slp_node);
10633 :
10634 261670 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10635 261670 : group_gap_adj = 0;
10636 :
10637 : /* VEC_NUM is the number of vect stmts to be created for this group. */
10638 261670 : grouped_load = false;
10639 : /* If an SLP permutation is from N elements to N elements,
10640 : and if one vector holds a whole number of N, we can load
10641 : the inputs to the permutation in the same way as an
10642 : unpermuted sequence. In other cases we need to load the
10643 : whole group, not only the number of vector stmts the
10644 : permutation result fits in. */
10645 261670 : unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
10646 261670 : if (nested_in_vect_loop)
10647 : /* We do not support grouped accesses in a nested loop,
10648 : instead the access is contiguous but it might be
10649 : permuted. No gap adjustment is needed though. */
10650 : ;
10651 261668 : else if (ls.slp_perm
10652 261668 : && (group_size != scalar_lanes
10653 10076 : || !multiple_p (nunits, group_size)))
10654 : {
10655 : /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10656 : variable VF; see vect_transform_slp_perm_load. */
10657 48582 : unsigned int const_vf = vf.to_constant ();
10658 48582 : unsigned int const_nunits = nunits.to_constant ();
10659 48582 : vec_num = CEIL (group_size * const_vf, const_nunits);
10660 48582 : group_gap_adj = vf * group_size - nunits * vec_num;
10661 : }
10662 : else
10663 : {
10664 213086 : group_gap_adj = group_size - scalar_lanes;
10665 : }
10666 :
10667 261670 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10668 : }
10669 : else
10670 : {
10671 330958 : first_stmt_info = stmt_info;
10672 330958 : first_dr_info = dr_info;
10673 330958 : group_size = 1;
10674 330958 : group_gap_adj = 0;
10675 330958 : ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
10676 : }
10677 :
10678 592628 : vec_loop_masks *loop_masks
10679 382342 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10680 592628 : ? &LOOP_VINFO_MASKS (loop_vinfo)
10681 38 : : NULL);
10682 38 : vec_loop_lens *loop_lens
10683 382342 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
10684 : ? &LOOP_VINFO_LENS (loop_vinfo)
10685 0 : : NULL);
10686 :
10687 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10688 : are some difference here. We cannot enable both the lens and masks
10689 : during transform but it is allowed during analysis.
10690 : Shouldn't go with length-based approach if fully masked. */
10691 592628 : if (cost_vec == NULL)
10692 : /* The cost_vec is NULL during transform. */
10693 153273 : gcc_assert ((!loop_lens || !loop_masks));
10694 :
10695 : /* Targets with store-lane instructions must not require explicit
10696 : realignment. vect_supportable_dr_alignment always returns either
10697 : dr_aligned or dr_unaligned_supported for (non-length) masked
10698 : operations. */
10699 592628 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
10700 : && !mask_node
10701 : && !loop_masks)
10702 : || mat_gather_scatter_p (memory_access_type)
10703 : || alignment_support_scheme == dr_aligned
10704 : || alignment_support_scheme == dr_unaligned_supported);
10705 :
10706 : /* In case the vectorization factor (VF) is bigger than the number
10707 : of elements that we can fit in a vectype (nunits), we have to generate
10708 : more than one vector stmt - i.e - we need to "unroll" the
10709 : vector stmt by a factor VF/nunits. In doing so, we record a pointer
10710 : from one copy of the vector stmt to the next, in the field
10711 : STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10712 : stages to find the correct vector defs to be used when vectorizing
10713 : stmts that use the defs of the current stmt. The example below
10714 : illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10715 : need to create 4 vectorized stmts):
10716 :
10717 : before vectorization:
10718 : RELATED_STMT VEC_STMT
10719 : S1: x = memref - -
10720 : S2: z = x + 1 - -
10721 :
10722 : step 1: vectorize stmt S1:
10723 : We first create the vector stmt VS1_0, and, as usual, record a
10724 : pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10725 : Next, we create the vector stmt VS1_1, and record a pointer to
10726 : it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10727 : Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10728 : stmts and pointers:
10729 : RELATED_STMT VEC_STMT
10730 : VS1_0: vx0 = memref0 VS1_1 -
10731 : VS1_1: vx1 = memref1 VS1_2 -
10732 : VS1_2: vx2 = memref2 VS1_3 -
10733 : VS1_3: vx3 = memref3 - -
10734 : S1: x = load - VS1_0
10735 : S2: z = x + 1 - -
10736 : */
10737 :
10738 : /* If the data reference is aligned (dr_aligned) or potentially unaligned
10739 : on a target that supports unaligned accesses (dr_unaligned_supported)
10740 : we generate the following code:
10741 : p = initial_addr;
10742 : indx = 0;
10743 : loop {
10744 : p = p + indx * vectype_size;
10745 : vec_dest = *(p);
10746 : indx = indx + 1;
10747 : }
10748 :
10749 : Otherwise, the data reference is potentially unaligned on a target that
10750 : does not support unaligned accesses (dr_explicit_realign_optimized) -
10751 : then generate the following code, in which the data in each iteration is
10752 : obtained by two vector loads, one from the previous iteration, and one
10753 : from the current iteration:
10754 : p1 = initial_addr;
10755 : msq_init = *(floor(p1))
10756 : p2 = initial_addr + VS - 1;
10757 : realignment_token = call target_builtin;
10758 : indx = 0;
10759 : loop {
10760 : p2 = p2 + indx * vectype_size
10761 : lsq = *(floor(p2))
10762 : vec_dest = realign_load (msq, lsq, realignment_token)
10763 : indx = indx + 1;
10764 : msq = lsq;
10765 : } */
10766 :
10767 : /* If the misalignment remains the same throughout the execution of the
10768 : loop, we can create the init_addr and permutation mask at the loop
10769 : preheader. Otherwise, it needs to be created inside the loop.
10770 : This can only occur when vectorizing memory accesses in the inner-loop
10771 : nested within an outer-loop that is being vectorized. */
10772 :
10773 592628 : if (nested_in_vect_loop
10774 592628 : && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
10775 1234 : GET_MODE_SIZE (TYPE_MODE (vectype))))
10776 : {
10777 195 : gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
10778 : compute_in_loop = true;
10779 : }
10780 :
10781 592628 : bool diff_first_stmt_info
10782 592628 : = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
10783 :
10784 592628 : tree offset = NULL_TREE;
10785 592628 : if ((alignment_support_scheme == dr_explicit_realign_optimized
10786 592628 : || alignment_support_scheme == dr_explicit_realign)
10787 0 : && !compute_in_loop)
10788 : {
10789 : /* If we have different first_stmt_info, we can't set up realignment
10790 : here, since we can't guarantee first_stmt_info DR has been
10791 : initialized yet, use first_stmt_info_for_drptr DR by bumping the
10792 : distance from first_stmt_info DR instead as below. */
10793 0 : if (!costing_p)
10794 : {
10795 0 : if (!diff_first_stmt_info)
10796 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
10797 : &realignment_token,
10798 : alignment_support_scheme, NULL_TREE,
10799 : &at_loop);
10800 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
10801 : {
10802 0 : phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
10803 0 : offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
10804 : size_one_node);
10805 0 : gcc_assert (!first_stmt_info_for_drptr);
10806 : }
10807 : }
10808 : }
10809 : else
10810 592628 : at_loop = loop;
10811 :
10812 592628 : if (!known_eq (poffset, 0))
10813 4781 : offset = (offset
10814 4781 : ? size_binop (PLUS_EXPR, offset, size_int (poffset))
10815 4781 : : size_int (poffset));
10816 :
10817 592628 : tree dr_increment;
10818 592628 : tree dr_bump;
10819 592628 : tree vec_offset = NULL_TREE;
10820 :
10821 592628 : auto_vec<tree> vec_offsets;
10822 592628 : auto_vec<tree> vec_masks;
10823 592628 : if (mask_node && !costing_p)
10824 630 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
10825 : &vec_masks);
10826 :
10827 592628 : tree vec_mask = NULL_TREE;
10828 592628 : tree vec_els = NULL_TREE;
10829 592628 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10830 : {
10831 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
10832 :
10833 0 : gcc_assert (alignment_support_scheme == dr_aligned
10834 : || alignment_support_scheme == dr_unaligned_supported);
10835 :
10836 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
10837 0 : if (!costing_p)
10838 : {
10839 0 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
10840 : memory_access_type);
10841 0 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
10842 : memory_access_type);
10843 : }
10844 :
10845 0 : unsigned int inside_cost = 0, prologue_cost = 0;
10846 : /* For costing some adjacent vector loads, we'd like to cost with
10847 : the total number of them once instead of cost each one by one. */
10848 0 : unsigned int n_adjacent_loads = 0;
10849 0 : int ncopies = vec_num / group_size;
10850 0 : for (j = 0; j < ncopies; j++)
10851 : {
10852 0 : if (costing_p)
10853 : {
10854 : /* An IFN_LOAD_LANES will load all its vector results,
10855 : regardless of which ones we actually need. Account
10856 : for the cost of unused results. */
10857 0 : if (first_stmt_info == stmt_info)
10858 : {
10859 0 : unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
10860 0 : stmt_vec_info next_stmt_info = first_stmt_info;
10861 0 : do
10862 : {
10863 0 : gaps -= 1;
10864 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
10865 : }
10866 0 : while (next_stmt_info);
10867 0 : if (gaps)
10868 : {
10869 0 : if (dump_enabled_p ())
10870 0 : dump_printf_loc (MSG_NOTE, vect_location,
10871 : "vect_model_load_cost: %d "
10872 : "unused vectors.\n",
10873 : gaps);
10874 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
10875 : alignment_support_scheme,
10876 : misalignment, false, &inside_cost,
10877 : &prologue_cost, cost_vec, cost_vec,
10878 : true);
10879 : }
10880 : }
10881 0 : n_adjacent_loads++;
10882 0 : continue;
10883 0 : }
10884 :
10885 : /* 1. Create the vector or array pointer update chain. */
10886 0 : if (j == 0)
10887 0 : dataref_ptr
10888 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10889 : at_loop, offset, &dummy, gsi,
10890 : NULL, false, dr_increment);
10891 : else
10892 : {
10893 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
10894 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10895 : stmt_info, dr_bump);
10896 : }
10897 0 : if (mask_node)
10898 0 : vec_mask = vec_masks[j];
10899 :
10900 0 : tree vec_array = create_vector_array (vectype, group_size);
10901 :
10902 0 : tree final_mask = NULL_TREE;
10903 0 : tree final_len = NULL_TREE;
10904 0 : tree bias = NULL_TREE;
10905 0 : if (loop_masks)
10906 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10907 : ncopies, vectype, j);
10908 0 : if (vec_mask)
10909 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
10910 : vec_mask, gsi);
10911 :
10912 0 : if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
10913 : {
10914 0 : if (loop_lens)
10915 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
10916 : ncopies, vectype, j, 1, true);
10917 : else
10918 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
10919 0 : signed char biasval
10920 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
10921 0 : bias = build_int_cst (intQI_type_node, biasval);
10922 0 : if (!final_mask)
10923 : {
10924 0 : mask_vectype = truth_type_for (vectype);
10925 0 : final_mask = build_minus_one_cst (mask_vectype);
10926 : }
10927 : }
10928 :
10929 0 : if (final_mask)
10930 : {
10931 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
10932 0 : if (type_mode_padding_p
10933 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
10934 0 : need_zeroing = true;
10935 : }
10936 :
10937 0 : gcall *call;
10938 0 : if (final_len && final_mask)
10939 : {
10940 : /* Emit:
10941 : VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10942 : VEC_MASK, LEN, BIAS). */
10943 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10944 0 : tree alias_ptr = build_int_cst (ref_type, align);
10945 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
10946 : dataref_ptr, alias_ptr,
10947 : final_mask, vec_els,
10948 : final_len, bias);
10949 : }
10950 0 : else if (final_mask)
10951 : {
10952 : /* Emit:
10953 : VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10954 : VEC_MASK). */
10955 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10956 0 : tree alias_ptr = build_int_cst (ref_type, align);
10957 0 : call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
10958 : dataref_ptr, alias_ptr,
10959 : final_mask, vec_els);
10960 : }
10961 : else
10962 : {
10963 : /* Emit:
10964 : VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10965 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
10966 0 : call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
10967 : }
10968 0 : gimple_call_set_lhs (call, vec_array);
10969 0 : gimple_call_set_nothrow (call, true);
10970 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
10971 :
10972 : /* Extract each vector into an SSA_NAME. */
10973 0 : for (unsigned i = 0; i < group_size; i++)
10974 : {
10975 0 : new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
10976 : vec_array, i, need_zeroing,
10977 : final_mask);
10978 0 : slp_node->push_vec_def (new_temp);
10979 : }
10980 :
10981 : /* Record that VEC_ARRAY is now dead. */
10982 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
10983 : }
10984 :
10985 0 : if (costing_p)
10986 : {
10987 0 : if (n_adjacent_loads > 0)
10988 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10989 : alignment_support_scheme, misalignment, false,
10990 : &inside_cost, &prologue_cost, cost_vec,
10991 : cost_vec, true);
10992 0 : if (dump_enabled_p ())
10993 0 : dump_printf_loc (MSG_NOTE, vect_location,
10994 : "vect_model_load_cost: inside_cost = %u, "
10995 : "prologue_cost = %u .\n",
10996 : inside_cost, prologue_cost);
10997 0 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10998 0 : slp_node->data = new vect_load_store_data (std::move (ls));
10999 : }
11000 :
11001 0 : return true;
11002 : }
11003 :
11004 592628 : if (mat_gather_scatter_p (memory_access_type))
11005 : {
11006 2990 : gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
11007 :
11008 2990 : auto_vec<tree> dr_chain (vec_num);
11009 :
11010 : /* If we pun the original vectype the loads as well as costing, length,
11011 : etc. is performed with the new type. After loading we VIEW_CONVERT
11012 : the data to the original vectype. */
11013 2990 : tree original_vectype = vectype;
11014 2990 : if (ls.ls_type)
11015 0 : vectype = ls.ls_type;
11016 :
11017 : /* 1. Create the vector or array pointer update chain. */
11018 2990 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11019 : {
11020 2990 : aggr_type = NULL_TREE;
11021 2990 : dr_increment = NULL_TREE;
11022 2990 : if (!costing_p)
11023 778 : vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
11024 : &vec_offsets);
11025 : }
11026 : else
11027 : {
11028 0 : aggr_type = elem_type;
11029 0 : if (!costing_p)
11030 : {
11031 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
11032 : ls.strided_offset_vectype,
11033 : loop_vinfo, gsi,
11034 : &dr_increment, &dr_bump,
11035 : &vec_offset);
11036 0 : dataref_ptr
11037 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11038 : at_loop, offset, &dummy, gsi,
11039 : NULL, false, dr_increment);
11040 : }
11041 : }
11042 :
11043 2990 : unsigned int inside_cost = 0, prologue_cost = 0;
11044 :
11045 2990 : gimple *new_stmt = NULL;
11046 6768 : for (i = 0; i < vec_num; i++)
11047 : {
11048 3778 : tree final_mask = NULL_TREE;
11049 3778 : tree final_len = NULL_TREE;
11050 3778 : tree bias = NULL_TREE;
11051 3778 : if (!costing_p)
11052 : {
11053 994 : if (mask_node)
11054 148 : vec_mask = vec_masks[i];
11055 994 : if (loop_masks)
11056 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11057 : vec_num, vectype, i);
11058 994 : if (vec_mask)
11059 148 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11060 : final_mask, vec_mask, gsi);
11061 :
11062 994 : if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11063 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11064 : stmt_info, dr_bump);
11065 : }
11066 :
11067 : /* 2. Create the vector-load in the loop. */
11068 3778 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
11069 3778 : tree alias_align_ptr = build_int_cst (ref_type, align);
11070 3778 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
11071 : {
11072 0 : if (costing_p)
11073 : {
11074 0 : if (ls.supported_offset_vectype
11075 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
11076 : vec_offset))
11077 0 : inside_cost
11078 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11079 : slp_node, 0, vect_body);
11080 0 : if (ls.supported_scale)
11081 0 : inside_cost
11082 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11083 : slp_node, 0, vect_body);
11084 :
11085 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11086 0 : inside_cost
11087 0 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11088 : slp_node, 0, vect_body);
11089 0 : continue;
11090 0 : }
11091 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11092 0 : vec_offset = vec_offsets[i];
11093 0 : tree zero = build_zero_cst (vectype);
11094 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11095 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
11096 :
11097 : /* Perform the offset conversion and scaling if necessary. */
11098 0 : if (!strided
11099 0 : && (ls.supported_offset_vectype || ls.supported_scale))
11100 : {
11101 0 : gimple_seq stmts = NULL;
11102 0 : if (ls.supported_offset_vectype)
11103 0 : vec_offset = gimple_convert
11104 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
11105 0 : if (ls.supported_scale)
11106 : {
11107 : /* Only scale the vec_offset if we haven't already. */
11108 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
11109 0 : || i == 0)
11110 : {
11111 0 : tree mult_cst = build_int_cst
11112 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
11113 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
11114 0 : tree mult = build_vector_from_val
11115 0 : (TREE_TYPE (vec_offset), mult_cst);
11116 0 : vec_offset = gimple_build
11117 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
11118 : vec_offset, mult);
11119 : }
11120 0 : scale = size_int (ls.supported_scale);
11121 : }
11122 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11123 : }
11124 :
11125 0 : if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
11126 : {
11127 0 : if (loop_lens)
11128 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11129 : vec_num, vectype, i, 1, true);
11130 : else
11131 0 : final_len = build_int_cst (sizetype,
11132 0 : TYPE_VECTOR_SUBPARTS (vectype));
11133 0 : signed char biasval
11134 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11135 0 : bias = build_int_cst (intQI_type_node, biasval);
11136 0 : if (!final_mask)
11137 : {
11138 0 : mask_vectype = truth_type_for (vectype);
11139 0 : final_mask = build_minus_one_cst (mask_vectype);
11140 : }
11141 : }
11142 :
11143 0 : if (final_mask)
11144 : {
11145 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11146 0 : if (type_mode_padding_p
11147 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11148 0 : need_zeroing = true;
11149 : }
11150 :
11151 0 : gcall *call;
11152 0 : if (final_len && final_mask)
11153 : {
11154 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
11155 0 : call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
11156 : 9, dataref_ptr,
11157 : alias_align_ptr,
11158 : vec_offset, scale, zero,
11159 : final_mask, vec_els,
11160 : final_len, bias);
11161 : else
11162 : /* Non-vector offset indicates that prefer to take
11163 : MASK_LEN_STRIDED_LOAD instead of the
11164 : MASK_LEN_GATHER_LOAD with direct stride arg. */
11165 0 : call = gimple_build_call_internal
11166 0 : (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
11167 : vec_offset, zero, final_mask, vec_els, final_len,
11168 : bias);
11169 : }
11170 0 : else if (final_mask)
11171 0 : call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
11172 : 7, dataref_ptr,
11173 : alias_align_ptr,
11174 : vec_offset, scale,
11175 : zero, final_mask, vec_els);
11176 : else
11177 0 : call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
11178 : dataref_ptr,
11179 : alias_align_ptr,
11180 : vec_offset, scale, zero);
11181 0 : gimple_call_set_nothrow (call, true);
11182 0 : new_stmt = call;
11183 0 : data_ref = NULL_TREE;
11184 : }
11185 3778 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
11186 : {
11187 : /* The builtin decls path for gather is legacy, x86 only. */
11188 828 : gcc_assert (!final_len && nunits.is_constant ());
11189 828 : if (costing_p)
11190 : {
11191 554 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11192 554 : inside_cost
11193 554 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11194 : slp_node, 0, vect_body);
11195 554 : continue;
11196 554 : }
11197 274 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11198 274 : poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
11199 274 : if (known_eq (nunits, offset_nunits))
11200 : {
11201 133 : new_stmt = vect_build_one_gather_load_call
11202 133 : (vinfo, stmt_info, slp_node, vectype, gsi,
11203 133 : ls.gs.decl, dataref_ptr, vec_offsets[i],
11204 : final_mask);
11205 133 : data_ref = NULL_TREE;
11206 : }
11207 141 : else if (known_eq (nunits, offset_nunits * 2))
11208 : {
11209 : /* We have a offset vector with half the number of
11210 : lanes but the builtins will produce full vectype
11211 : data with just the lower lanes filled. */
11212 63 : new_stmt = vect_build_one_gather_load_call
11213 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11214 63 : ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
11215 : final_mask);
11216 63 : tree low = make_ssa_name (vectype);
11217 63 : gimple_set_lhs (new_stmt, low);
11218 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11219 :
11220 : /* now put upper half of final_mask in final_mask low. */
11221 63 : if (final_mask
11222 63 : && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
11223 : {
11224 11 : int count = nunits.to_constant ();
11225 11 : vec_perm_builder sel (count, count, 1);
11226 11 : sel.quick_grow (count);
11227 98 : for (int i = 0; i < count; ++i)
11228 76 : sel[i] = i | (count / 2);
11229 11 : vec_perm_indices indices (sel, 2, count);
11230 11 : tree perm_mask = vect_gen_perm_mask_checked
11231 11 : (TREE_TYPE (final_mask), indices);
11232 11 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11233 : final_mask, final_mask,
11234 : perm_mask);
11235 11 : final_mask = make_ssa_name (TREE_TYPE (final_mask));
11236 11 : gimple_set_lhs (new_stmt, final_mask);
11237 11 : vect_finish_stmt_generation (vinfo, stmt_info,
11238 : new_stmt, gsi);
11239 11 : }
11240 52 : else if (final_mask)
11241 : {
11242 24 : new_stmt = gimple_build_assign (NULL_TREE,
11243 : VEC_UNPACK_HI_EXPR,
11244 : final_mask);
11245 24 : final_mask = make_ssa_name
11246 24 : (truth_type_for (offset_vectype));
11247 24 : gimple_set_lhs (new_stmt, final_mask);
11248 24 : vect_finish_stmt_generation (vinfo, stmt_info,
11249 : new_stmt, gsi);
11250 : }
11251 :
11252 63 : new_stmt = vect_build_one_gather_load_call
11253 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11254 : ls.gs.decl, dataref_ptr,
11255 63 : vec_offsets[2 * i + 1], final_mask);
11256 63 : tree high = make_ssa_name (vectype);
11257 63 : gimple_set_lhs (new_stmt, high);
11258 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11259 :
11260 : /* compose low + high. */
11261 63 : int count = nunits.to_constant ();
11262 63 : vec_perm_builder sel (count, count, 1);
11263 63 : sel.quick_grow (count);
11264 710 : for (int i = 0; i < count; ++i)
11265 584 : sel[i] = i < count / 2 ? i : i + count / 2;
11266 63 : vec_perm_indices indices (sel, 2, count);
11267 63 : tree perm_mask
11268 63 : = vect_gen_perm_mask_checked (vectype, indices);
11269 63 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11270 : low, high, perm_mask);
11271 63 : data_ref = NULL_TREE;
11272 63 : }
11273 78 : else if (known_eq (nunits * 2, offset_nunits))
11274 : {
11275 : /* We have a offset vector with double the number of
11276 : lanes. Select the low/high part accordingly. */
11277 78 : vec_offset = vec_offsets[i / 2];
11278 78 : if (i & 1)
11279 : {
11280 39 : int count = offset_nunits.to_constant ();
11281 39 : vec_perm_builder sel (count, count, 1);
11282 39 : sel.quick_grow (count);
11283 474 : for (int i = 0; i < count; ++i)
11284 396 : sel[i] = i | (count / 2);
11285 39 : vec_perm_indices indices (sel, 2, count);
11286 39 : tree perm_mask = vect_gen_perm_mask_checked
11287 39 : (TREE_TYPE (vec_offset), indices);
11288 39 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11289 : vec_offset, vec_offset,
11290 : perm_mask);
11291 39 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
11292 39 : gimple_set_lhs (new_stmt, vec_offset);
11293 39 : vect_finish_stmt_generation (vinfo, stmt_info,
11294 : new_stmt, gsi);
11295 39 : }
11296 78 : new_stmt = vect_build_one_gather_load_call
11297 78 : (vinfo, stmt_info, slp_node, vectype, gsi,
11298 : ls.gs.decl,
11299 : dataref_ptr, vec_offset, final_mask);
11300 78 : data_ref = NULL_TREE;
11301 : }
11302 : else
11303 0 : gcc_unreachable ();
11304 : }
11305 : else
11306 : {
11307 : /* Emulated gather-scatter. */
11308 2950 : gcc_assert (!final_mask);
11309 2950 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
11310 2950 : if (costing_p)
11311 : {
11312 : /* For emulated gathers N offset vector element
11313 : offset add is consumed by the load). */
11314 2230 : inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
11315 : slp_node, 0, vect_body);
11316 : /* N scalar loads plus gathering them into a
11317 : vector. */
11318 2230 : inside_cost
11319 2230 : = record_stmt_cost (cost_vec, const_nunits, scalar_load,
11320 : slp_node, 0, vect_body);
11321 2230 : inside_cost
11322 2230 : = record_stmt_cost (cost_vec, 1, vec_construct,
11323 : slp_node, 0, vect_body);
11324 2230 : continue;
11325 : }
11326 720 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11327 720 : unsigned HOST_WIDE_INT const_offset_nunits
11328 720 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
11329 720 : vec<constructor_elt, va_gc> *ctor_elts;
11330 720 : vec_alloc (ctor_elts, const_nunits);
11331 720 : gimple_seq stmts = NULL;
11332 : /* We support offset vectors with more elements
11333 : than the data vector for now. */
11334 720 : unsigned HOST_WIDE_INT factor
11335 : = const_offset_nunits / const_nunits;
11336 720 : vec_offset = vec_offsets[i / factor];
11337 720 : unsigned elt_offset = (i % factor) * const_nunits;
11338 720 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
11339 720 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11340 720 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
11341 3616 : for (unsigned k = 0; k < const_nunits; ++k)
11342 : {
11343 2176 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
11344 : bitsize_int (k + elt_offset));
11345 6528 : tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
11346 2176 : vec_offset, TYPE_SIZE (idx_type),
11347 : boff);
11348 2176 : idx = gimple_convert (&stmts, sizetype, idx);
11349 2176 : idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
11350 2176 : tree ptr = gimple_build (&stmts, PLUS_EXPR,
11351 2176 : TREE_TYPE (dataref_ptr),
11352 : dataref_ptr, idx);
11353 2176 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
11354 2176 : tree elt = make_ssa_name (TREE_TYPE (vectype));
11355 2176 : tree ref = build2 (MEM_REF, ltype, ptr,
11356 : build_int_cst (ref_type, 0));
11357 2176 : new_stmt = gimple_build_assign (elt, ref);
11358 4352 : gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
11359 2176 : gimple_seq_add_stmt (&stmts, new_stmt);
11360 2176 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
11361 : }
11362 720 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11363 720 : new_stmt = gimple_build_assign (NULL_TREE,
11364 : build_constructor (vectype,
11365 : ctor_elts));
11366 720 : data_ref = NULL_TREE;
11367 : }
11368 :
11369 994 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11370 : /* DATA_REF is null if we've already built the statement. */
11371 994 : if (data_ref)
11372 : {
11373 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11374 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11375 : }
11376 1988 : new_temp = (need_zeroing
11377 994 : ? make_ssa_name (vectype)
11378 994 : : make_ssa_name (vec_dest, new_stmt));
11379 994 : gimple_set_lhs (new_stmt, new_temp);
11380 994 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11381 :
11382 : /* If we need to explicitly zero inactive elements emit a
11383 : VEC_COND_EXPR that does so. */
11384 994 : if (need_zeroing)
11385 : {
11386 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11387 : vectype);
11388 :
11389 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11390 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11391 : final_mask, new_temp, vec_els);
11392 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11393 0 : new_temp = new_temp2;
11394 : }
11395 :
11396 994 : if (ls.ls_type)
11397 : {
11398 0 : new_stmt = gimple_build_assign (make_ssa_name
11399 : (original_vectype),
11400 : VIEW_CONVERT_EXPR,
11401 : build1 (VIEW_CONVERT_EXPR,
11402 : original_vectype,
11403 : new_temp));
11404 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11405 : }
11406 :
11407 : /* Store vector loads in the corresponding SLP_NODE. */
11408 994 : if (!costing_p)
11409 : {
11410 994 : if (ls.slp_perm)
11411 0 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
11412 : else
11413 994 : slp_node->push_vec_def (new_stmt);
11414 : }
11415 : }
11416 :
11417 2990 : if (ls.slp_perm)
11418 : {
11419 0 : if (costing_p)
11420 : {
11421 0 : gcc_assert (ls.n_perms != -1U);
11422 0 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11423 : slp_node, 0, vect_body);
11424 : }
11425 : else
11426 : {
11427 0 : unsigned n_perms2;
11428 0 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
11429 : false, &n_perms2);
11430 0 : gcc_assert (ls.n_perms == n_perms2);
11431 : }
11432 : }
11433 :
11434 2990 : if (costing_p)
11435 : {
11436 2212 : if (dump_enabled_p ())
11437 315 : dump_printf_loc (MSG_NOTE, vect_location,
11438 : "vect_model_load_cost: inside_cost = %u, "
11439 : "prologue_cost = %u .\n",
11440 : inside_cost, prologue_cost);
11441 2212 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11442 2212 : slp_node->data = new vect_load_store_data (std::move (ls));
11443 : }
11444 2990 : return true;
11445 2990 : }
11446 :
11447 589638 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
11448 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
11449 :
11450 589638 : aggr_type = vectype;
11451 589638 : if (!costing_p)
11452 : {
11453 152495 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
11454 : memory_access_type);
11455 152495 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
11456 : memory_access_type);
11457 : }
11458 :
11459 589638 : poly_uint64 group_elt = 0;
11460 589638 : unsigned int inside_cost = 0, prologue_cost = 0;
11461 : /* For costing some adjacent vector loads, we'd like to cost with
11462 : the total number of them once instead of cost each one by one. */
11463 589638 : unsigned int n_adjacent_loads = 0;
11464 :
11465 : /* 1. Create the vector or array pointer update chain. */
11466 589638 : if (!costing_p)
11467 : {
11468 152495 : bool simd_lane_access_p
11469 152495 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
11470 152495 : if (simd_lane_access_p
11471 1634 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
11472 1634 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
11473 1634 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
11474 1634 : && integer_zerop (DR_INIT (first_dr_info->dr))
11475 1634 : && alias_sets_conflict_p (get_alias_set (aggr_type),
11476 1634 : get_alias_set (TREE_TYPE (ref_type)))
11477 152495 : && (alignment_support_scheme == dr_aligned
11478 1634 : || alignment_support_scheme == dr_unaligned_supported))
11479 : {
11480 1634 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
11481 1634 : dataref_offset = build_int_cst (ref_type, 0);
11482 : }
11483 150861 : else if (diff_first_stmt_info)
11484 : {
11485 3435 : dataref_ptr
11486 3435 : = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
11487 : aggr_type, at_loop, offset, &dummy,
11488 : gsi, NULL, simd_lane_access_p,
11489 : dr_increment);
11490 : /* Adjust the pointer by the difference to first_stmt. */
11491 3435 : data_reference_p ptrdr
11492 : = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
11493 3435 : tree diff = fold_convert (sizetype,
11494 : size_binop (MINUS_EXPR,
11495 : DR_INIT (first_dr_info->dr),
11496 : DR_INIT (ptrdr)));
11497 3435 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11498 : stmt_info, diff);
11499 3435 : if (alignment_support_scheme == dr_explicit_realign)
11500 : {
11501 0 : msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
11502 : vectype, gsi,
11503 : &realignment_token,
11504 : alignment_support_scheme,
11505 : dataref_ptr, &at_loop);
11506 0 : gcc_assert (!compute_in_loop);
11507 : }
11508 : }
11509 : else
11510 147426 : dataref_ptr
11511 147426 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11512 : at_loop,
11513 : offset, &dummy, gsi, NULL,
11514 : simd_lane_access_p, dr_increment);
11515 : }
11516 :
11517 589638 : auto_vec<tree> dr_chain;
11518 589638 : if (grouped_load || ls.slp_perm)
11519 57476 : dr_chain.create (vec_num);
11520 :
11521 589638 : gimple *new_stmt = NULL;
11522 1541659 : for (i = 0; i < vec_num; i++)
11523 : {
11524 952021 : tree final_mask = NULL_TREE;
11525 952021 : tree final_len = NULL_TREE;
11526 952021 : tree bias = NULL_TREE;
11527 :
11528 952021 : if (!costing_p)
11529 : {
11530 240864 : if (mask_node)
11531 661 : vec_mask = vec_masks[i];
11532 240864 : if (loop_masks)
11533 58 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11534 : vec_num, vectype, i);
11535 240864 : if (vec_mask)
11536 661 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11537 : final_mask, vec_mask, gsi);
11538 :
11539 240864 : if (i > 0)
11540 88369 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11541 : dr_bump);
11542 : }
11543 :
11544 : /* 2. Create the vector-load in the loop. */
11545 952021 : switch (alignment_support_scheme)
11546 : {
11547 952021 : case dr_aligned:
11548 952021 : case dr_unaligned_supported:
11549 952021 : {
11550 952021 : if (costing_p)
11551 : break;
11552 :
11553 240864 : unsigned int misalign;
11554 240864 : unsigned HOST_WIDE_INT align;
11555 240864 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
11556 240864 : if (alignment_support_scheme == dr_aligned)
11557 : misalign = 0;
11558 156479 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
11559 : {
11560 119071 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
11561 119071 : misalign = 0;
11562 : }
11563 : else
11564 37408 : misalign = misalignment;
11565 240864 : if (dataref_offset == NULL_TREE
11566 238732 : && TREE_CODE (dataref_ptr) == SSA_NAME)
11567 172765 : set_ssa_name_alignment (dataref_ptr, align, misalign);
11568 240864 : align = least_bit_hwi (misalign | align);
11569 :
11570 : /* Compute IFN when LOOP_LENS or final_mask valid. */
11571 240864 : machine_mode vmode = TYPE_MODE (vectype);
11572 240864 : machine_mode new_vmode = vmode;
11573 240864 : internal_fn partial_ifn = IFN_LAST;
11574 240864 : if (loop_lens)
11575 : {
11576 0 : opt_machine_mode new_ovmode
11577 0 : = get_len_load_store_mode (vmode, true, &partial_ifn);
11578 0 : new_vmode = new_ovmode.require ();
11579 0 : unsigned factor
11580 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
11581 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11582 : vec_num, vectype, i, factor, true);
11583 : }
11584 240864 : else if (final_mask)
11585 : {
11586 699 : if (!can_vec_mask_load_store_p (vmode,
11587 699 : TYPE_MODE
11588 : (TREE_TYPE (final_mask)),
11589 : true, &partial_ifn))
11590 0 : gcc_unreachable ();
11591 : }
11592 :
11593 240864 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11594 : {
11595 0 : if (!final_len)
11596 : {
11597 : /* Pass VF value to 'len' argument of
11598 : MASK_LEN_LOAD if LOOP_LENS is invalid. */
11599 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11600 : }
11601 0 : if (!final_mask)
11602 : {
11603 : /* Pass all ones value to 'mask' argument of
11604 : MASK_LEN_LOAD if final_mask is invalid. */
11605 0 : mask_vectype = truth_type_for (vectype);
11606 0 : final_mask = build_minus_one_cst (mask_vectype);
11607 : }
11608 : }
11609 240864 : if (final_len)
11610 : {
11611 0 : signed char biasval
11612 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11613 0 : bias = build_int_cst (intQI_type_node, biasval);
11614 : }
11615 :
11616 0 : tree vec_els;
11617 :
11618 0 : if (final_len)
11619 : {
11620 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11621 0 : gcall *call;
11622 :
11623 : /* Need conversion if the vectype is punned by VnQI. */
11624 0 : els_vectype = vectype;
11625 0 : if (vmode != new_vmode)
11626 0 : els_vectype
11627 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11628 : new_vmode);
11629 0 : vec_els = vect_get_mask_load_else (maskload_elsval,
11630 : els_vectype);
11631 :
11632 0 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11633 : {
11634 0 : if (type_mode_padding_p
11635 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11636 0 : need_zeroing = true;
11637 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
11638 : 6, dataref_ptr, ptr,
11639 : final_mask, vec_els,
11640 : final_len, bias);
11641 : }
11642 : else
11643 0 : call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
11644 : dataref_ptr, ptr,
11645 : vec_els, final_len,
11646 : bias);
11647 0 : gimple_call_set_nothrow (call, true);
11648 0 : new_stmt = call;
11649 0 : data_ref = NULL_TREE;
11650 :
11651 : /* Need conversion if it's wrapped with VnQI. */
11652 0 : if (vmode != new_vmode)
11653 : {
11654 0 : tree new_vtype
11655 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11656 : new_vmode);
11657 0 : tree var = vect_get_new_ssa_name (new_vtype,
11658 : vect_simple_var);
11659 0 : gimple_set_lhs (call, var);
11660 0 : vect_finish_stmt_generation (vinfo, stmt_info, call,
11661 : gsi);
11662 0 : tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
11663 0 : new_stmt = gimple_build_assign (vec_dest,
11664 : VIEW_CONVERT_EXPR, op);
11665 : }
11666 : }
11667 240864 : else if (final_mask)
11668 : {
11669 699 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11670 699 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11671 699 : if (type_mode_padding_p
11672 699 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11673 0 : need_zeroing = true;
11674 699 : gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
11675 : dataref_ptr, ptr,
11676 : final_mask,
11677 : vec_els);
11678 699 : gimple_call_set_nothrow (call, true);
11679 699 : new_stmt = call;
11680 699 : data_ref = NULL_TREE;
11681 : }
11682 : else
11683 : {
11684 240165 : tree ltype = vectype;
11685 240165 : tree new_vtype = NULL_TREE;
11686 240165 : unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
11687 240165 : unsigned HOST_WIDE_INT dr_size
11688 240165 : = vect_get_scalar_dr_size (first_dr_info);
11689 240165 : poly_int64 off = 0;
11690 240165 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11691 1488 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
11692 240165 : unsigned int vect_align
11693 240165 : = vect_known_alignment_in_bytes (first_dr_info, vectype,
11694 240165 : off);
11695 : /* Try to use a single smaller load when we are about
11696 : to load excess elements compared to the unrolled
11697 : scalar loop. */
11698 240165 : if (known_gt ((i + 1) * nunits,
11699 : (group_size * vf - gap)))
11700 : {
11701 7035 : poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
11702 7035 : if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
11703 : nunits))
11704 : /* DR will be unused. */
11705 : ltype = NULL_TREE;
11706 2381 : else if (known_ge (vect_align,
11707 : tree_to_poly_uint64
11708 : (TYPE_SIZE_UNIT (vectype))))
11709 : /* Aligned access to excess elements is OK if
11710 : at least one element is accessed in the
11711 : scalar loop. */
11712 : ;
11713 1977 : else if (known_gt (vect_align,
11714 : ((nunits - remain) * dr_size)))
11715 : /* Aligned access to the gap area when there's
11716 : at least one element in it is OK. */
11717 : ;
11718 : else
11719 : {
11720 : /* remain should now be > 0 and < nunits. */
11721 1972 : unsigned num;
11722 1972 : if (known_ne (remain, 0u)
11723 1972 : && constant_multiple_p (nunits, remain, &num))
11724 : {
11725 1480 : tree ptype;
11726 1480 : new_vtype
11727 1480 : = vector_vector_composition_type (vectype, num,
11728 : &ptype);
11729 1480 : if (new_vtype)
11730 1480 : ltype = ptype;
11731 : }
11732 : /* Else use multiple loads or a masked load? */
11733 : /* For loop vectorization we now should have
11734 : an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11735 : set. */
11736 1972 : if (loop_vinfo)
11737 1730 : gcc_assert (new_vtype
11738 : || LOOP_VINFO_PEELING_FOR_GAPS
11739 : (loop_vinfo));
11740 : /* But still reduce the access size to the next
11741 : required power-of-two so peeling a single
11742 : scalar iteration is sufficient. */
11743 1972 : unsigned HOST_WIDE_INT cremain;
11744 1972 : if (remain.is_constant (&cremain))
11745 : {
11746 1972 : unsigned HOST_WIDE_INT cpart_size
11747 1972 : = 1 << ceil_log2 (cremain);
11748 1972 : if (known_gt (nunits, cpart_size)
11749 1972 : && constant_multiple_p (nunits, cpart_size,
11750 : &num))
11751 : {
11752 1492 : tree ptype;
11753 1492 : new_vtype
11754 2984 : = vector_vector_composition_type (vectype,
11755 1492 : num,
11756 : &ptype);
11757 1492 : if (new_vtype)
11758 1492 : ltype = ptype;
11759 : }
11760 : }
11761 : }
11762 : }
11763 240165 : tree offset = (dataref_offset ? dataref_offset
11764 238033 : : build_int_cst (ref_type, 0));
11765 240165 : if (!ltype)
11766 : ;
11767 235511 : else if (ltype != vectype
11768 235511 : && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11769 : {
11770 21 : poly_uint64 gap_offset
11771 21 : = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
11772 21 : - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
11773 21 : tree gapcst = build_int_cstu (ref_type, gap_offset);
11774 21 : offset = size_binop (PLUS_EXPR, offset, gapcst);
11775 : }
11776 235511 : if (ltype)
11777 : {
11778 235511 : data_ref = fold_build2 (MEM_REF, ltype,
11779 : dataref_ptr, offset);
11780 235511 : if (alignment_support_scheme == dr_aligned
11781 235511 : && align >= TYPE_ALIGN_UNIT (ltype))
11782 : ;
11783 : else
11784 154793 : TREE_TYPE (data_ref)
11785 309586 : = build_aligned_type (TREE_TYPE (data_ref),
11786 : align * BITS_PER_UNIT);
11787 : }
11788 235511 : if (!ltype)
11789 4654 : data_ref = build_constructor (vectype, NULL);
11790 235511 : else if (ltype != vectype)
11791 : {
11792 1492 : vect_copy_ref_info (data_ref,
11793 1492 : DR_REF (first_dr_info->dr));
11794 1492 : tree tem = make_ssa_name (ltype);
11795 1492 : new_stmt = gimple_build_assign (tem, data_ref);
11796 1492 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11797 : gsi);
11798 1492 : data_ref = NULL;
11799 1492 : vec<constructor_elt, va_gc> *v;
11800 : /* We've computed 'num' above to statically two
11801 : or via constant_multiple_p. */
11802 1492 : unsigned num
11803 1492 : = (exact_div (tree_to_poly_uint64
11804 1492 : (TYPE_SIZE_UNIT (vectype)),
11805 : tree_to_poly_uint64
11806 1492 : (TYPE_SIZE_UNIT (ltype)))
11807 1492 : .to_constant ());
11808 1492 : vec_alloc (v, num);
11809 1492 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11810 : {
11811 54 : while (--num)
11812 54 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11813 : build_zero_cst (ltype));
11814 21 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11815 : }
11816 : else
11817 : {
11818 1471 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11819 1471 : while (--num)
11820 3384 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11821 : build_zero_cst (ltype));
11822 : }
11823 1492 : gcc_assert (new_vtype != NULL_TREE);
11824 1492 : if (new_vtype == vectype)
11825 1460 : new_stmt
11826 1460 : = gimple_build_assign (vec_dest,
11827 : build_constructor (vectype, v));
11828 : else
11829 : {
11830 32 : tree new_vname = make_ssa_name (new_vtype);
11831 32 : new_stmt
11832 32 : = gimple_build_assign (new_vname,
11833 : build_constructor (new_vtype,
11834 : v));
11835 32 : vect_finish_stmt_generation (vinfo, stmt_info,
11836 : new_stmt, gsi);
11837 32 : new_stmt
11838 32 : = gimple_build_assign (vec_dest,
11839 : build1 (VIEW_CONVERT_EXPR,
11840 : vectype, new_vname));
11841 : }
11842 : }
11843 : }
11844 : break;
11845 : }
11846 0 : case dr_explicit_realign:
11847 0 : {
11848 0 : if (costing_p)
11849 : break;
11850 0 : tree ptr, bump;
11851 :
11852 0 : tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11853 :
11854 0 : if (compute_in_loop)
11855 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
11856 : gsi, &realignment_token,
11857 : dr_explicit_realign,
11858 : dataref_ptr, NULL);
11859 :
11860 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11861 0 : ptr = copy_ssa_name (dataref_ptr);
11862 : else
11863 0 : ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
11864 : // For explicit realign the target alignment should be
11865 : // known at compile time.
11866 0 : unsigned HOST_WIDE_INT align
11867 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11868 0 : new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
11869 : build_int_cst
11870 0 : (TREE_TYPE (dataref_ptr),
11871 0 : -(HOST_WIDE_INT) align));
11872 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11873 0 : data_ref = build2 (MEM_REF, vectype,
11874 : ptr, build_int_cst (ref_type, 0));
11875 0 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11876 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11877 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11878 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11879 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11880 0 : gimple_move_vops (new_stmt, stmt_info->stmt);
11881 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11882 0 : msq = new_temp;
11883 :
11884 0 : bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
11885 0 : bump = size_binop (MINUS_EXPR, bump, size_one_node);
11886 0 : ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11887 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
11888 0 : build_int_cst (TREE_TYPE (ptr),
11889 0 : -(HOST_WIDE_INT) align));
11890 0 : if (TREE_CODE (ptr) == SSA_NAME)
11891 0 : ptr = copy_ssa_name (ptr, new_stmt);
11892 : else
11893 0 : ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
11894 0 : gimple_assign_set_lhs (new_stmt, ptr);
11895 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11896 0 : data_ref = build2 (MEM_REF, vectype,
11897 : ptr, build_int_cst (ref_type, 0));
11898 0 : break;
11899 : }
11900 0 : case dr_explicit_realign_optimized:
11901 0 : {
11902 0 : if (costing_p)
11903 : break;
11904 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11905 0 : new_temp = copy_ssa_name (dataref_ptr);
11906 : else
11907 0 : new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
11908 : // We should only be doing this if we know the target
11909 : // alignment at compile time.
11910 0 : unsigned HOST_WIDE_INT align
11911 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11912 0 : new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
11913 0 : build_int_cst (TREE_TYPE (dataref_ptr),
11914 0 : -(HOST_WIDE_INT) align));
11915 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11916 0 : data_ref = build2 (MEM_REF, vectype, new_temp,
11917 : build_int_cst (ref_type, 0));
11918 0 : break;
11919 : }
11920 0 : default:
11921 0 : gcc_unreachable ();
11922 : }
11923 :
11924 : /* One common place to cost the above vect load for different
11925 : alignment support schemes. */
11926 952021 : if (costing_p)
11927 : {
11928 : /* For the prologue cost for realign,
11929 : we only need to count it once for the whole group. */
11930 711157 : bool first_stmt_info_p = first_stmt_info == stmt_info;
11931 711157 : bool add_realign_cost = first_stmt_info_p && i == 0;
11932 711157 : if (memory_access_type == VMAT_CONTIGUOUS
11933 711157 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11934 : {
11935 : /* Leave realign cases alone to keep them simple. */
11936 711157 : if (alignment_support_scheme == dr_explicit_realign_optimized
11937 : || alignment_support_scheme == dr_explicit_realign)
11938 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
11939 : alignment_support_scheme, misalignment,
11940 : add_realign_cost, &inside_cost,
11941 : &prologue_cost, cost_vec, cost_vec,
11942 : true);
11943 : else
11944 711157 : n_adjacent_loads++;
11945 : }
11946 : }
11947 : else
11948 : {
11949 240864 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11950 : /* DATA_REF is null if we've already built the statement. */
11951 240864 : if (data_ref)
11952 : {
11953 238673 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11954 238673 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11955 : }
11956 :
11957 481728 : new_temp = (need_zeroing
11958 240864 : ? make_ssa_name (vectype)
11959 240864 : : make_ssa_name (vec_dest, new_stmt));
11960 240864 : gimple_set_lhs (new_stmt, new_temp);
11961 240864 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11962 :
11963 : /* If we need to explicitly zero inactive elements emit a
11964 : VEC_COND_EXPR that does so. */
11965 240864 : if (need_zeroing)
11966 : {
11967 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11968 : vectype);
11969 :
11970 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11971 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11972 : final_mask, new_temp, vec_els);
11973 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11974 : gsi);
11975 0 : new_temp = new_temp2;
11976 : }
11977 : }
11978 :
11979 : /* 3. Handle explicit realignment if necessary/supported.
11980 : Create in loop:
11981 : vec_dest = realign_load (msq, lsq, realignment_token) */
11982 952021 : if (!costing_p
11983 240864 : && (alignment_support_scheme == dr_explicit_realign_optimized
11984 : || alignment_support_scheme == dr_explicit_realign))
11985 : {
11986 0 : lsq = gimple_assign_lhs (new_stmt);
11987 0 : if (!realignment_token)
11988 0 : realignment_token = dataref_ptr;
11989 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11990 0 : new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
11991 : lsq, realignment_token);
11992 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11993 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11994 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11995 :
11996 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
11997 : {
11998 0 : gcc_assert (phi);
11999 0 : if (i == vec_num - 1)
12000 0 : add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
12001 : UNKNOWN_LOCATION);
12002 : msq = lsq;
12003 : }
12004 : }
12005 :
12006 952021 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
12007 : {
12008 6089 : if (costing_p)
12009 4601 : inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
12010 : slp_node, 0, vect_body);
12011 : else
12012 : {
12013 1488 : tree perm_mask = perm_mask_for_reverse (vectype);
12014 1488 : new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
12015 : perm_mask, stmt_info, gsi);
12016 1488 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
12017 : }
12018 : }
12019 :
12020 : /* Collect vector loads and later create their permutation in
12021 : vect_transform_slp_perm_load. */
12022 952021 : if (!costing_p && (grouped_load || ls.slp_perm))
12023 73735 : dr_chain.quick_push (new_temp);
12024 :
12025 : /* Store vector loads in the corresponding SLP_NODE. */
12026 240864 : if (!costing_p && !ls.slp_perm)
12027 167129 : slp_node->push_vec_def (new_stmt);
12028 :
12029 : /* With SLP permutation we load the gaps as well, without
12030 : we need to skip the gaps after we manage to fully load
12031 : all elements. group_gap_adj is DR_GROUP_SIZE here. */
12032 952021 : group_elt += nunits;
12033 952021 : if (!costing_p
12034 240864 : && maybe_ne (group_gap_adj, 0U)
12035 40410 : && !ls.slp_perm
12036 967496 : && known_eq (group_elt, group_size - group_gap_adj))
12037 : {
12038 12802 : poly_wide_int bump_val
12039 12802 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
12040 12802 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
12041 0 : bump_val = -bump_val;
12042 12802 : tree bump = wide_int_to_tree (sizetype, bump_val);
12043 12802 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
12044 : bump);
12045 12802 : group_elt = 0;
12046 12802 : }
12047 : }
12048 : /* Bump the vector pointer to account for a gap or for excess
12049 : elements loaded for a permuted SLP load. */
12050 589638 : if (!costing_p
12051 152495 : && maybe_ne (group_gap_adj, 0U)
12052 602908 : && ls.slp_perm)
12053 : {
12054 468 : poly_wide_int bump_val
12055 468 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
12056 468 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
12057 9 : bump_val = -bump_val;
12058 468 : tree bump = wide_int_to_tree (sizetype, bump_val);
12059 468 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
12060 468 : }
12061 :
12062 589638 : if (ls.slp_perm)
12063 : {
12064 : /* For SLP we know we've seen all possible uses of dr_chain so
12065 : direct vect_transform_slp_perm_load to DCE the unused parts.
12066 : ??? This is a hack to prevent compile-time issues as seen
12067 : in PR101120 and friends. */
12068 57476 : if (costing_p)
12069 : {
12070 40576 : gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
12071 40576 : if (ls.n_perms != 0)
12072 40042 : inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
12073 : slp_node, 0, vect_body);
12074 40576 : if (n_adjacent_loads > 0)
12075 40576 : n_adjacent_loads = ls.n_loads;
12076 : }
12077 : else
12078 : {
12079 16900 : unsigned n_perms2, n_loads2;
12080 16900 : bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
12081 : gsi, vf, false, &n_perms2,
12082 : &n_loads2, true);
12083 16900 : gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
12084 : }
12085 : }
12086 :
12087 589638 : if (costing_p)
12088 : {
12089 437143 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
12090 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
12091 437143 : if (n_adjacent_loads > 0)
12092 437143 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
12093 : alignment_support_scheme, misalignment, false,
12094 : &inside_cost, &prologue_cost, cost_vec, cost_vec,
12095 : true);
12096 437143 : if (dump_enabled_p ())
12097 25930 : dump_printf_loc (MSG_NOTE, vect_location,
12098 : "vect_model_load_cost: inside_cost = %u, "
12099 : "prologue_cost = %u .\n",
12100 : inside_cost, prologue_cost);
12101 437143 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
12102 437143 : slp_node->data = new vect_load_store_data (std::move (ls));
12103 : }
12104 :
12105 589638 : return true;
12106 1331606 : }
12107 :
12108 : /* vectorizable_condition.
12109 :
12110 : Check if STMT_INFO is conditional modify expression that can be vectorized.
12111 : If COST_VEC is passed, calculate costs but don't change anything,
12112 : otherwise, vectorize STMT_INFO: create a vectorized stmt using
12113 : VEC_COND_EXPR to replace it, and insert it at GSI.
12114 :
12115 : When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12116 :
12117 : Return true if STMT_INFO is vectorizable in this way. */
12118 :
12119 : static bool
12120 739869 : vectorizable_condition (vec_info *vinfo,
12121 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12122 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12123 : {
12124 739869 : tree scalar_dest = NULL_TREE;
12125 739869 : tree vec_dest = NULL_TREE;
12126 739869 : tree then_clause, else_clause;
12127 739869 : tree vec_cond_lhs = NULL_TREE;
12128 739869 : tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
12129 739869 : tree vec_compare;
12130 739869 : tree new_temp;
12131 739869 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
12132 739869 : enum vect_def_type dts[4]
12133 : = {vect_unknown_def_type, vect_unknown_def_type,
12134 : vect_unknown_def_type, vect_unknown_def_type};
12135 739869 : enum tree_code code;
12136 739869 : int i;
12137 739869 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12138 739869 : vec<tree> vec_oprnds0 = vNULL;
12139 739869 : vec<tree> vec_oprnds2 = vNULL;
12140 739869 : vec<tree> vec_oprnds3 = vNULL;
12141 739869 : tree vec_cmp_type;
12142 :
12143 739869 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12144 : return false;
12145 :
12146 : /* Is vectorizable conditional operation? */
12147 739869 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12148 421212 : if (!stmt)
12149 : return false;
12150 :
12151 421212 : code = gimple_assign_rhs_code (stmt);
12152 421212 : if (code != COND_EXPR)
12153 : return false;
12154 :
12155 35965 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
12156 35965 : vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
12157 35965 : bool nested_cycle_p = false;
12158 35965 : bool for_reduction = vect_is_reduction (stmt_info);
12159 35965 : if (for_reduction)
12160 : {
12161 648 : if (SLP_TREE_LANES (slp_node) > 1)
12162 : return false;
12163 : /* ??? With a reduction path we do not get at the reduction info from
12164 : every stmt, use the conservative default setting then. */
12165 729 : if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
12166 : {
12167 630 : vect_reduc_info reduc_info
12168 630 : = info_for_reduction (loop_vinfo, slp_node);
12169 630 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
12170 630 : nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12171 : stmt_info);
12172 : }
12173 : }
12174 : else
12175 : {
12176 35317 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12177 : return false;
12178 : }
12179 :
12180 35965 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12181 35965 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12182 :
12183 35965 : int vec_num = vect_get_num_copies (vinfo, slp_node);
12184 :
12185 35965 : slp_tree slp_cond;
12186 35965 : tree cond_expr = gimple_assign_rhs1 (stmt);
12187 35965 : gcc_assert (! COMPARISON_CLASS_P (cond_expr));
12188 35965 : if (TREE_CODE (cond_expr) != SSA_NAME
12189 35965 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond_expr))
12190 35965 : || !vect_is_simple_use (vinfo, slp_node, 0,
12191 : &slp_cond, &dts[0], &vec_cmp_type)
12192 35965 : || !vec_cmp_type
12193 71915 : || !VECTOR_BOOLEAN_TYPE_P (vec_cmp_type))
12194 : return false;
12195 :
12196 35950 : slp_tree then_slp_node, else_slp_node;
12197 35950 : if (!vect_is_simple_use (vinfo, slp_node, 1,
12198 : &then_clause, &then_slp_node, &dts[2], &vectype1))
12199 : return false;
12200 35950 : if (!vect_is_simple_use (vinfo, slp_node, 2,
12201 : &else_clause, &else_slp_node, &dts[3], &vectype2))
12202 : return false;
12203 :
12204 35950 : if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
12205 : return false;
12206 :
12207 35950 : if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
12208 : return false;
12209 :
12210 35938 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
12211 71876 : TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
12212 : return false;
12213 :
12214 : /* For conditional reductions, the "then" value needs to be the candidate
12215 : value calculated by this iteration while the "else" value needs to be
12216 : the result carried over from previous iterations. If the COND_EXPR
12217 : is the other way around, we need to swap it. */
12218 35938 : bool must_invert_cmp_result = false;
12219 35938 : if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
12220 : {
12221 739869 : must_invert_cmp_result = true;
12222 : /* ??? The vectorized operand query below doesn't allow swapping
12223 : this way for SLP. */
12224 : return false;
12225 : /* std::swap (then_clause, else_clause); */
12226 : }
12227 :
12228 35938 : if (cost_vec)
12229 : {
12230 27199 : vect_cost_for_stmt kind = vector_stmt;
12231 27199 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12232 : /* Count one reduction-like operation per vector. */
12233 : kind = vec_to_scalar;
12234 27199 : else if (!expand_vec_cond_expr_p (vectype, vec_cmp_type))
12235 : return false;
12236 :
12237 27185 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
12238 : vec_cmp_type)
12239 27185 : || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
12240 54370 : || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
12241 : {
12242 0 : if (dump_enabled_p ())
12243 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12244 : "incompatible vector types for invariants\n");
12245 : return false;
12246 : }
12247 :
12248 27185 : if (loop_vinfo && for_reduction
12249 472 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12250 : {
12251 81 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12252 : {
12253 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12254 : vectype, OPTIMIZE_FOR_SPEED))
12255 0 : vect_record_loop_len (loop_vinfo,
12256 : &LOOP_VINFO_LENS (loop_vinfo),
12257 : vec_num, vectype, 1);
12258 : else
12259 0 : vect_record_loop_mask (loop_vinfo,
12260 : &LOOP_VINFO_MASKS (loop_vinfo),
12261 : vec_num, vectype, NULL);
12262 : }
12263 : /* Extra inactive lanes should be safe for vect_nested_cycle. */
12264 81 : else if (!nested_cycle_p)
12265 : {
12266 81 : if (dump_enabled_p ())
12267 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12268 : "conditional reduction prevents the use"
12269 : " of partial vectors.\n");
12270 81 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
12271 : }
12272 : }
12273 :
12274 27185 : SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
12275 27185 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
12276 27185 : return true;
12277 : }
12278 :
12279 : /* Transform. */
12280 :
12281 : /* Handle def. */
12282 8739 : scalar_dest = gimple_assign_lhs (stmt);
12283 8739 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12284 8739 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
12285 :
12286 8739 : bool swap_cond_operands = false;
12287 :
12288 : /* See whether another part of the vectorized code applies a loop
12289 : mask to the condition, or to its inverse. */
12290 :
12291 8739 : vec_loop_masks *masks = NULL;
12292 8739 : vec_loop_lens *lens = NULL;
12293 8739 : if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12294 : {
12295 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12296 0 : lens = &LOOP_VINFO_LENS (loop_vinfo);
12297 : }
12298 8739 : else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
12299 : {
12300 3 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12301 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12302 : else
12303 : {
12304 3 : scalar_cond_masked_key cond (cond_expr, 1);
12305 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12306 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12307 : else
12308 : {
12309 3 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
12310 3 : tree_code orig_code = cond.code;
12311 3 : cond.code = invert_tree_comparison (cond.code, honor_nans);
12312 : /* Try the inverse of the current mask. We check if the
12313 : inverse mask is live and if so we generate a negate of
12314 : the current mask such that we still honor NaNs. */
12315 3 : cond.inverted_p = true;
12316 3 : cond.code = orig_code;
12317 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12318 : {
12319 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12320 0 : swap_cond_operands = true;
12321 0 : must_invert_cmp_result = true;
12322 : }
12323 : }
12324 : }
12325 : }
12326 :
12327 : /* Handle cond expr. */
12328 8739 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds2,
12329 : reduction_type != EXTRACT_LAST_REDUCTION, &vec_oprnds3);
12330 :
12331 8739 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12332 0 : vec_else_clause = else_clause;
12333 :
12334 : /* Arguments are ready. Create the new vector stmt. */
12335 20311 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
12336 : {
12337 11572 : vec_then_clause = vec_oprnds2[i];
12338 11572 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12339 11572 : vec_else_clause = vec_oprnds3[i];
12340 :
12341 11572 : if (swap_cond_operands)
12342 0 : std::swap (vec_then_clause, vec_else_clause);
12343 :
12344 11572 : vec_compare = vec_cond_lhs;
12345 :
12346 : /* If we decided to apply a loop mask to the result of the vector
12347 : comparison, AND the comparison with the mask now. Later passes
12348 : should then be able to reuse the AND results between multiple
12349 : vector statements.
12350 :
12351 : For example:
12352 : for (int i = 0; i < 100; ++i)
12353 : x[i] = y[i] ? z[i] : 10;
12354 :
12355 : results in following optimized GIMPLE:
12356 :
12357 : mask__35.8_43 = vect__4.7_41 != { 0, ... };
12358 : vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12359 : _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12360 : vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12361 : vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12362 : vect_iftmp.11_47, { 10, ... }>;
12363 :
12364 : instead of using a masked and unmasked forms of
12365 : vec != { 0, ... } (masked in the MASK_LOAD,
12366 : unmasked in the VEC_COND_EXPR). */
12367 :
12368 : /* Force vec_compare to be an SSA_NAME rather than a comparison,
12369 : in cases where that's necessary. */
12370 :
12371 11572 : tree len = NULL_TREE, bias = NULL_TREE;
12372 11572 : if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
12373 : {
12374 0 : if (!is_gimple_val (vec_compare))
12375 : {
12376 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12377 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12378 : vec_compare);
12379 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12380 0 : vec_compare = vec_compare_name;
12381 : }
12382 :
12383 0 : if (must_invert_cmp_result)
12384 : {
12385 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12386 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12387 : BIT_NOT_EXPR,
12388 : vec_compare);
12389 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12390 0 : vec_compare = vec_compare_name;
12391 : }
12392 :
12393 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12394 : vectype, OPTIMIZE_FOR_SPEED))
12395 : {
12396 0 : if (lens)
12397 : {
12398 : /* ??? Do we really want the adjusted LEN here? Isn't this
12399 : based on number of elements? */
12400 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens,
12401 : vec_num, vectype, i, 1, true);
12402 0 : signed char biasval
12403 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
12404 0 : bias = build_int_cst (intQI_type_node, biasval);
12405 : }
12406 : else
12407 : {
12408 0 : len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
12409 0 : bias = build_int_cst (intQI_type_node, 0);
12410 : }
12411 : }
12412 0 : if (masks)
12413 : {
12414 0 : tree loop_mask
12415 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12416 : vectype, i);
12417 0 : tree tmp2 = make_ssa_name (vec_cmp_type);
12418 0 : gassign *g
12419 0 : = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
12420 : loop_mask);
12421 0 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
12422 0 : vec_compare = tmp2;
12423 : }
12424 : }
12425 :
12426 0 : gimple *new_stmt;
12427 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12428 : {
12429 0 : gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
12430 0 : tree lhs = gimple_get_lhs (old_stmt);
12431 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12432 0 : lhs = copy_ssa_name (lhs);
12433 0 : if (len)
12434 0 : new_stmt = gimple_build_call_internal
12435 0 : (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
12436 : vec_then_clause, len, bias);
12437 : else
12438 0 : new_stmt = gimple_build_call_internal
12439 0 : (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
12440 : vec_then_clause);
12441 0 : gimple_call_set_lhs (new_stmt, lhs);
12442 0 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12443 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12444 : {
12445 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12446 0 : vec_else_clause = lhs;
12447 : }
12448 0 : else if (old_stmt == gsi_stmt (*gsi))
12449 0 : vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
12450 : else
12451 : {
12452 : /* In this case we're moving the definition to later in the
12453 : block. That doesn't matter because the only uses of the
12454 : lhs are in phi statements. */
12455 0 : gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
12456 0 : gsi_remove (&old_gsi, true);
12457 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12458 : }
12459 : }
12460 : else
12461 : {
12462 11572 : new_temp = make_ssa_name (vec_dest);
12463 11572 : new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
12464 : vec_then_clause, vec_else_clause);
12465 11572 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12466 : }
12467 11572 : slp_node->push_vec_def (new_stmt);
12468 : }
12469 :
12470 8739 : vec_oprnds0.release ();
12471 8739 : vec_oprnds2.release ();
12472 8739 : vec_oprnds3.release ();
12473 :
12474 8739 : return true;
12475 : }
12476 :
12477 : /* Helper of vectorizable_comparison.
12478 :
12479 : Check if STMT_INFO is comparison expression CODE that can be vectorized.
12480 : If COST_VEC is passed, calculate costs but don't change anything,
12481 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12482 : it at GSI.
12483 :
12484 : Return true if STMT_INFO is vectorizable in this way. */
12485 :
12486 : static bool
12487 393668 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
12488 : stmt_vec_info stmt_info, tree_code code,
12489 : gimple_stmt_iterator *gsi,
12490 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12491 : {
12492 393668 : tree lhs, rhs1, rhs2;
12493 393668 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12494 393668 : tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
12495 393668 : tree new_temp;
12496 393668 : enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
12497 393668 : poly_uint64 nunits;
12498 393668 : enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12499 393668 : int i;
12500 393668 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12501 393668 : vec<tree> vec_oprnds0 = vNULL;
12502 393668 : vec<tree> vec_oprnds1 = vNULL;
12503 393668 : tree mask_type;
12504 393668 : tree mask = NULL_TREE;
12505 :
12506 393668 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12507 : return false;
12508 :
12509 393668 : if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
12510 : return false;
12511 :
12512 173106 : mask_type = vectype;
12513 173106 : nunits = TYPE_VECTOR_SUBPARTS (vectype);
12514 :
12515 173106 : if (TREE_CODE_CLASS (code) != tcc_comparison)
12516 : return false;
12517 :
12518 171170 : slp_tree slp_rhs1, slp_rhs2;
12519 171170 : if (!vect_is_simple_use (vinfo, slp_node,
12520 : 0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
12521 : return false;
12522 :
12523 171170 : if (!vect_is_simple_use (vinfo, slp_node,
12524 : 1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
12525 : return false;
12526 :
12527 133088 : if (vectype1 && vectype2
12528 249405 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12529 78235 : TYPE_VECTOR_SUBPARTS (vectype2)))
12530 16 : return false;
12531 :
12532 171154 : vectype = vectype1 ? vectype1 : vectype2;
12533 :
12534 : /* Invariant comparison. */
12535 171154 : if (!vectype)
12536 : {
12537 33494 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
12538 33494 : if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
12539 : return false;
12540 : }
12541 137660 : else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
12542 : return false;
12543 :
12544 : /* Can't compare mask and non-mask types. */
12545 133072 : if (vectype1 && vectype2
12546 405175 : && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
12547 : return false;
12548 :
12549 : /* We cannot compare non-mode precision _BitInt types. Unlike bool
12550 : or bit-precision INTEGER_TYPE the padding bit values are target
12551 : dependent and possibly undefined. */
12552 171142 : if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE
12553 171142 : && !type_has_mode_precision_p (TREE_TYPE (rhs1)))
12554 : return false;
12555 :
12556 : /* Boolean values may have another representation in vectors
12557 : and therefore we prefer bit operations over comparison for
12558 : them (which also works for scalar masks). We store opcodes
12559 : to use in bitop1 and bitop2. Statement is vectorized as
12560 : BITOP2 (rhs1 BITOP1 rhs2) or
12561 : rhs1 BITOP2 (BITOP1 rhs2)
12562 : depending on bitop1 and bitop2 arity. */
12563 171076 : bool swap_p = false;
12564 171076 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
12565 : {
12566 765 : if (code == GT_EXPR)
12567 : {
12568 : bitop1 = BIT_NOT_EXPR;
12569 : bitop2 = BIT_AND_EXPR;
12570 : }
12571 : else if (code == GE_EXPR)
12572 : {
12573 : bitop1 = BIT_NOT_EXPR;
12574 : bitop2 = BIT_IOR_EXPR;
12575 : }
12576 : else if (code == LT_EXPR)
12577 : {
12578 : bitop1 = BIT_NOT_EXPR;
12579 : bitop2 = BIT_AND_EXPR;
12580 : swap_p = true;
12581 : }
12582 : else if (code == LE_EXPR)
12583 : {
12584 : bitop1 = BIT_NOT_EXPR;
12585 : bitop2 = BIT_IOR_EXPR;
12586 : swap_p = true;
12587 : }
12588 : else
12589 : {
12590 : bitop1 = BIT_XOR_EXPR;
12591 : if (code == EQ_EXPR)
12592 : bitop2 = BIT_NOT_EXPR;
12593 : }
12594 : }
12595 :
12596 171076 : if (cost_vec)
12597 : {
12598 158259 : if (bitop1 == NOP_EXPR)
12599 : {
12600 157643 : if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
12601 : return false;
12602 : }
12603 : else
12604 : {
12605 616 : machine_mode mode = TYPE_MODE (vectype);
12606 616 : optab optab;
12607 :
12608 616 : optab = optab_for_tree_code (bitop1, vectype, optab_default);
12609 616 : if (!optab || !can_implement_p (optab, mode))
12610 : return false;
12611 :
12612 616 : if (bitop2 != NOP_EXPR)
12613 : {
12614 97 : optab = optab_for_tree_code (bitop2, vectype, optab_default);
12615 97 : if (!optab || !can_implement_p (optab, mode))
12616 : return false;
12617 : }
12618 : }
12619 :
12620 : /* Put types on constant and invariant SLP children. */
12621 150299 : if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
12622 150299 : || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
12623 : {
12624 2 : if (dump_enabled_p ())
12625 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12626 : "incompatible vector types for invariants\n");
12627 : return false;
12628 : }
12629 :
12630 150297 : vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
12631 : slp_node, cost_vec);
12632 150297 : return true;
12633 : }
12634 :
12635 : /* Transform. */
12636 :
12637 : /* Handle def. */
12638 12817 : lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
12639 12817 : if (lhs)
12640 12817 : mask = vect_create_destination_var (lhs, mask_type);
12641 :
12642 12817 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds1);
12643 12817 : if (swap_p)
12644 60 : std::swap (vec_oprnds0, vec_oprnds1);
12645 :
12646 : /* Arguments are ready. Create the new vector stmt. */
12647 32154 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
12648 : {
12649 19337 : gimple *new_stmt;
12650 19337 : vec_rhs2 = vec_oprnds1[i];
12651 :
12652 19337 : if (lhs)
12653 19337 : new_temp = make_ssa_name (mask);
12654 : else
12655 0 : new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
12656 19337 : if (bitop1 == NOP_EXPR)
12657 : {
12658 19178 : new_stmt = gimple_build_assign (new_temp, code,
12659 : vec_rhs1, vec_rhs2);
12660 19178 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12661 : }
12662 : else
12663 : {
12664 159 : if (bitop1 == BIT_NOT_EXPR)
12665 86 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
12666 : else
12667 73 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
12668 : vec_rhs2);
12669 159 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12670 159 : if (bitop2 != NOP_EXPR)
12671 : {
12672 86 : tree res = make_ssa_name (mask);
12673 86 : if (bitop2 == BIT_NOT_EXPR)
12674 0 : new_stmt = gimple_build_assign (res, bitop2, new_temp);
12675 : else
12676 86 : new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
12677 : new_temp);
12678 86 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12679 : }
12680 : }
12681 19337 : slp_node->push_vec_def (new_stmt);
12682 : }
12683 :
12684 12817 : vec_oprnds0.release ();
12685 12817 : vec_oprnds1.release ();
12686 :
12687 12817 : return true;
12688 : }
12689 :
12690 : /* vectorizable_comparison.
12691 :
12692 : Check if STMT_INFO is comparison expression that can be vectorized.
12693 : If COST_VEC is passed, calculate costs but don't change anything,
12694 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12695 : it at GSI.
12696 :
12697 : Return true if STMT_INFO is vectorizable in this way. */
12698 :
12699 : static bool
12700 716762 : vectorizable_comparison (vec_info *vinfo,
12701 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12702 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12703 : {
12704 716762 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12705 :
12706 716762 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12707 : return false;
12708 :
12709 716762 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12710 : return false;
12711 :
12712 465191 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12713 390907 : if (!stmt)
12714 : return false;
12715 :
12716 390907 : enum tree_code code = gimple_assign_rhs_code (stmt);
12717 390907 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12718 390907 : if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
12719 : slp_node, cost_vec))
12720 : return false;
12721 :
12722 160353 : if (cost_vec)
12723 147536 : SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
12724 :
12725 : return true;
12726 : }
12727 :
12728 : /* Check to see if the target supports any of the compare and branch optabs for
12729 : vectors with MODE as these would be required when expanding. */
12730 : static bool
12731 67203 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
12732 : {
12733 67203 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12734 67203 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12735 :
12736 : /* The vectorizer only produces vec_cbranch_any_optab directly. So only
12737 : check for support for that or vec_cbranch_any_optab when masked.
12738 : We can't produce vcond_cbranch_any directly from the vectorizer as we
12739 : want to keep gimple_cond as the GIMPLE representation. But we'll fold
12740 : it in expand. For that reason we require a backend to support the
12741 : unconditional vector cbranch optab if they support the conditional one,
12742 : which is just an optimization on the unconditional one. */
12743 67203 : if (masked_loop_p
12744 67203 : && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
12745 : != CODE_FOR_nothing)
12746 : return true;
12747 67203 : else if (len_loop_p
12748 67203 : && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
12749 : != CODE_FOR_nothing)
12750 : return true;
12751 67203 : else if (!masked_loop_p && !len_loop_p
12752 134406 : && direct_optab_handler (vec_cbranch_any_optab, mode)
12753 : != CODE_FOR_nothing)
12754 : return true;
12755 :
12756 : /* The target can implement cbranch to distinguish between boolean vector
12757 : types and data types if they don't have a different mode for both. */
12758 67203 : return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
12759 : }
12760 :
12761 : /* Determine the type to use for early break vectorization's scalar IV. If
12762 : no type is possible return false. */
12763 :
12764 : static bool
12765 2761 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
12766 : {
12767 : /* Check if we have a usable scalar IV type for vectorization. */
12768 2761 : tree iters_vf_type = sizetype;
12769 2761 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
12770 : {
12771 : /* Find the type with the minimum precision we can use
12772 : for the scalar IV. */
12773 2536 : tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
12774 :
12775 : /* Work out how many bits we need to represent the limit. */
12776 2536 : unsigned int min_ni_width
12777 2536 : = vect_min_prec_for_max_niters (loop_vinfo, 1);
12778 :
12779 : /* Check if we're using PFA, if so we need a signed IV and an
12780 : extra bit for the sign. */
12781 2536 : if (TYPE_UNSIGNED (cand_type)
12782 2536 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
12783 4111 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12784 159 : min_ni_width += 1;
12785 :
12786 2536 : if (TYPE_PRECISION (cand_type) >= min_ni_width)
12787 2463 : iters_vf_type = unsigned_type_for (cand_type);
12788 : else
12789 : {
12790 73 : opt_scalar_int_mode cmp_mode_iter;
12791 73 : tree iv_type = NULL_TREE;
12792 357 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
12793 : {
12794 357 : auto cmp_mode = cmp_mode_iter.require ();
12795 357 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
12796 357 : if (cmp_bits >= min_ni_width
12797 357 : && targetm.scalar_mode_supported_p (cmp_mode))
12798 : {
12799 73 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
12800 73 : if (iv_type)
12801 : break;
12802 : }
12803 : }
12804 :
12805 73 : if (!iv_type)
12806 : {
12807 0 : if (dump_enabled_p ())
12808 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12809 : "can't vectorize early exit because the "
12810 : "target doesn't support a scalar type wide "
12811 : "wide enough to hold niters.\n");
12812 0 : return false;
12813 : }
12814 73 : iters_vf_type = iv_type;
12815 : }
12816 : }
12817 :
12818 2761 : LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
12819 2761 : return true;
12820 : }
12821 :
12822 : /* Check to see if the current early break given in STMT_INFO is valid for
12823 : vectorization. */
12824 :
12825 : bool
12826 274776 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
12827 : gimple_stmt_iterator *gsi,
12828 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12829 : {
12830 274776 : if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
12831 : return false;
12832 :
12833 68860 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
12834 : return false;
12835 :
12836 68860 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
12837 : return false;
12838 :
12839 68860 : DUMP_VECT_SCOPE ("vectorizable_early_exit");
12840 :
12841 68860 : auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
12842 :
12843 : /* For SLP we don't want to use the type of the operands of the SLP node, when
12844 : vectorizing using SLP slp_node will be the children of the gcond and we
12845 : want to use the type of the direct children which since the gcond is root
12846 : will be the current node, rather than a child node as vect_is_simple_use
12847 : assumes. */
12848 68860 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12849 68860 : if (!vectype)
12850 : return false;
12851 :
12852 68860 : machine_mode mode = TYPE_MODE (vectype);
12853 68860 : int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
12854 :
12855 68860 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
12856 68860 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
12857 68860 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12858 68860 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12859 :
12860 : /* Now build the new conditional. Pattern gimple_conds get dropped during
12861 : codegen so we must replace the original insn. */
12862 68860 : gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
12863 68860 : gcond *cond_stmt = as_a <gcond *>(orig_stmt);
12864 :
12865 68860 : tree vectype_out = vectype;
12866 68860 : auto bb = gimple_bb (cond_stmt);
12867 68860 : edge exit_true_edge = EDGE_SUCC (bb, 0);
12868 68860 : if (exit_true_edge->flags & EDGE_FALSE_VALUE)
12869 867 : exit_true_edge = EDGE_SUCC (bb, 1);
12870 68860 : gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
12871 :
12872 : /* When vectorizing we assume that if the branch edge is taken that we're
12873 : exiting the loop. This is not however always the case as the compiler will
12874 : rewrite conditions to always be a comparison against 0. To do this it
12875 : sometimes flips the edges. This is fine for scalar, but for vector we
12876 : then have to negate the result of the test, as we're still assuming that if
12877 : you take the branch edge that we found the exit condition. i.e. we need to
12878 : know whether we are generating a `forall` or an `exist` condition. */
12879 137720 : bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12880 68860 : exit_true_edge->dest);
12881 :
12882 : /* See if we support ADDHN and use that for the reduction. */
12883 68860 : internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
12884 68860 : bool addhn_supported_p
12885 68860 : = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
12886 68860 : tree narrow_type = NULL_TREE;
12887 68860 : if (addhn_supported_p)
12888 : {
12889 : /* Calculate the narrowing type for the result. */
12890 0 : auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
12891 0 : auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
12892 0 : tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
12893 0 : tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
12894 0 : narrow_type = truth_type_for (tmp_type);
12895 :
12896 0 : if (!supports_vector_compare_and_branch (loop_vinfo,
12897 0 : TYPE_MODE (narrow_type)))
12898 : {
12899 0 : if (dump_enabled_p ())
12900 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12901 : "can't use ADDHN reduction because cbranch for "
12902 : "the narrowed type is not supported by the "
12903 : "target.\n");
12904 : addhn_supported_p = false;
12905 : }
12906 : }
12907 :
12908 : /* Analyze only. */
12909 68860 : if (cost_vec)
12910 : {
12911 67203 : if (!addhn_supported_p
12912 67203 : && !supports_vector_compare_and_branch (loop_vinfo, mode))
12913 : {
12914 64442 : if (dump_enabled_p ())
12915 793 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12916 : "can't vectorize early exit because the "
12917 : "target doesn't support flag setting vector "
12918 : "comparisons.\n");
12919 : return false;
12920 : }
12921 :
12922 2761 : if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
12923 : slp_node, cost_vec))
12924 : return false;
12925 :
12926 2761 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12927 : {
12928 1575 : if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
12929 : OPTIMIZE_FOR_SPEED))
12930 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
12931 : else
12932 1575 : vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
12933 : }
12934 :
12935 2761 : if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
12936 : return false;
12937 :
12938 : return true;
12939 : }
12940 :
12941 : /* Transform. */
12942 :
12943 1657 : tree new_temp = NULL_TREE;
12944 1657 : gimple *new_stmt = NULL;
12945 :
12946 1657 : if (dump_enabled_p ())
12947 463 : dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
12948 :
12949 : /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
12950 : roots and so by the time we get to them we have already codegened the SLP tree
12951 : and so we shouldn't try to do so again. The arguments have already been
12952 : vectorized. It's not very clean to do this here, But the masking code below is
12953 : complex and this keeps it all in one place to ease fixes and backports. Once we
12954 : drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
12955 :
12956 1657 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
12957 1657 : basic_block cond_bb = gimple_bb (stmt);
12958 1657 : gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
12959 :
12960 1657 : auto_vec<tree> stmts;
12961 1657 : stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
12962 :
12963 : /* If we're comparing against a previous forall we need to negate the results
12964 : before we do the final comparison or reduction. */
12965 1657 : if (flipped)
12966 : {
12967 : /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
12968 : if (any(~mask)) by negating the masks and flipping the branches.
12969 :
12970 : 1. For unmasked loops we simply reduce the ~mask.
12971 : 2. For masked loops we reduce (~mask & loop_mask) which is the same as
12972 : doing (mask & loop_mask) ^ loop_mask. */
12973 329 : for (unsigned i = 0; i < stmts.length (); i++)
12974 : {
12975 199 : tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
12976 199 : auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
12977 199 : vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
12978 : &cond_gsi);
12979 199 : stmts[i] = inv_lhs;
12980 : }
12981 :
12982 130 : EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12983 130 : EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12984 : }
12985 :
12986 : /* Determine if we need to reduce the final value. */
12987 1657 : if (stmts.length () > 1)
12988 : {
12989 : /* We build the reductions in a way to maintain as much parallelism as
12990 : possible. */
12991 151 : auto_vec<tree> workset (stmts.length ());
12992 :
12993 : /* Mask the statements as we queue them up. Normally we loop over
12994 : vec_num, but since we inspect the exact results of vectorization
12995 : we don't need to and instead can just use the stmts themselves. */
12996 151 : if (masked_loop_p)
12997 0 : for (unsigned i = 0; i < stmts.length (); i++)
12998 : {
12999 0 : tree stmt_mask
13000 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
13001 : vectype, i);
13002 0 : stmt_mask
13003 0 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
13004 0 : stmts[i], &cond_gsi);
13005 0 : workset.quick_push (stmt_mask);
13006 : }
13007 151 : else if (len_loop_p)
13008 151 : for (unsigned i = 0; i < stmts.length (); i++)
13009 : {
13010 0 : tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
13011 : lens, vec_num,
13012 0 : vectype, stmts[i], i, 1);
13013 :
13014 0 : workset.quick_push (len_mask);
13015 : }
13016 : else
13017 151 : workset.splice (stmts);
13018 :
13019 454 : while (workset.length () > 1)
13020 : {
13021 303 : tree arg0 = workset.pop ();
13022 303 : tree arg1 = workset.pop ();
13023 303 : if (addhn_supported_p && workset.length () == 0)
13024 : {
13025 0 : new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
13026 0 : vectype_out = narrow_type;
13027 0 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
13028 0 : gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
13029 0 : gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
13030 : }
13031 : else
13032 : {
13033 303 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
13034 303 : new_stmt
13035 303 : = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
13036 : }
13037 303 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
13038 : &cond_gsi);
13039 303 : workset.quick_insert (0, new_temp);
13040 : }
13041 151 : }
13042 : else
13043 : {
13044 1506 : new_temp = stmts[0];
13045 1506 : if (masked_loop_p)
13046 : {
13047 2 : tree mask
13048 2 : = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
13049 2 : new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
13050 : new_temp, &cond_gsi);
13051 : }
13052 1504 : else if (len_loop_p)
13053 0 : new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
13054 : 1, vectype, new_temp, 0, 1);
13055 : }
13056 :
13057 1657 : gcc_assert (new_temp);
13058 :
13059 1657 : tree cst = build_zero_cst (vectype_out);
13060 1657 : gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
13061 1657 : update_stmt (orig_stmt);
13062 :
13063 : /* ??? */
13064 1657 : SLP_TREE_VEC_DEFS (slp_node).truncate (0);
13065 :
13066 1657 : return true;
13067 1657 : }
13068 :
13069 : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13070 : can handle all live statements in the node. Otherwise return true
13071 : if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13072 : VEC_STMT_P is as for vectorizable_live_operation. */
13073 :
13074 : static bool
13075 1339089 : can_vectorize_live_stmts (vec_info *vinfo,
13076 : slp_tree slp_node, slp_instance slp_node_instance,
13077 : bool vec_stmt_p,
13078 : stmt_vector_for_cost *cost_vec)
13079 : {
13080 1339089 : stmt_vec_info slp_stmt_info;
13081 1339089 : unsigned int i;
13082 2815002 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
13083 : {
13084 1475913 : if (slp_stmt_info
13085 1459355 : && STMT_VINFO_LIVE_P (slp_stmt_info)
13086 1614410 : && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13087 : slp_node_instance, i,
13088 : vec_stmt_p, cost_vec))
13089 : return false;
13090 : }
13091 :
13092 : return true;
13093 : }
13094 :
13095 : /* Make sure the statement is vectorizable. */
13096 :
13097 : opt_result
13098 2673960 : vect_analyze_stmt (vec_info *vinfo,
13099 : slp_tree node, slp_instance node_instance,
13100 : stmt_vector_for_cost *cost_vec)
13101 : {
13102 2673960 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
13103 2673960 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
13104 2673960 : enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
13105 2673960 : bool ok;
13106 :
13107 2673960 : if (dump_enabled_p ())
13108 107900 : dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
13109 : stmt_info->stmt);
13110 :
13111 5042820 : if (gimple_has_volatile_ops (stmt_info->stmt))
13112 : {
13113 : /* ??? This shouldn't really happen, volatile stmts should
13114 : not end up in the SLP graph. */
13115 0 : return opt_result::failure_at (stmt_info->stmt,
13116 : "not vectorized:"
13117 : " stmt has volatile operands: %G\n",
13118 : stmt_info->stmt);
13119 : }
13120 :
13121 : /* Skip stmts that do not need to be vectorized. */
13122 2673960 : if (!STMT_VINFO_RELEVANT_P (stmt_info)
13123 0 : && !STMT_VINFO_LIVE_P (stmt_info))
13124 : {
13125 0 : if (dump_enabled_p ())
13126 0 : dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
13127 :
13128 : /* ??? This shouldn't really happen, irrelevant stmts should
13129 : not end up in the SLP graph. */
13130 0 : return opt_result::failure_at (stmt_info->stmt,
13131 : "not vectorized:"
13132 : " irrelevant stmt as SLP node %p "
13133 : "representative.\n",
13134 : (void *)node);
13135 : }
13136 :
13137 2673960 : switch (STMT_VINFO_DEF_TYPE (stmt_info))
13138 : {
13139 : case vect_internal_def:
13140 : case vect_condition_def:
13141 : break;
13142 :
13143 82404 : case vect_reduction_def:
13144 82404 : case vect_nested_cycle:
13145 82404 : gcc_assert (!bb_vinfo
13146 : && (relevance == vect_used_in_outer
13147 : || relevance == vect_used_in_outer_by_reduction
13148 : || relevance == vect_used_by_reduction
13149 : || relevance == vect_unused_in_scope
13150 : || relevance == vect_used_only_live));
13151 : break;
13152 :
13153 314 : case vect_double_reduction_def:
13154 314 : gcc_assert (!bb_vinfo && node);
13155 : break;
13156 :
13157 168847 : case vect_induction_def:
13158 168847 : case vect_first_order_recurrence:
13159 168847 : gcc_assert (!bb_vinfo);
13160 : break;
13161 :
13162 0 : case vect_constant_def:
13163 0 : case vect_external_def:
13164 0 : case vect_unknown_def_type:
13165 0 : default:
13166 0 : gcc_unreachable ();
13167 : }
13168 :
13169 2673960 : tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
13170 2673960 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13171 :
13172 2673960 : if (STMT_VINFO_RELEVANT_P (stmt_info))
13173 : {
13174 2673960 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
13175 2673960 : gcc_assert (SLP_TREE_VECTYPE (node)
13176 : || gimple_code (stmt_info->stmt) == GIMPLE_COND
13177 : || (call && gimple_call_lhs (call) == NULL_TREE));
13178 : }
13179 :
13180 2673960 : ok = true;
13181 2673960 : if (bb_vinfo
13182 1545005 : || (STMT_VINFO_RELEVANT_P (stmt_info)
13183 0 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
13184 : /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13185 : -mveclibabi= takes preference over library functions with
13186 : the simd attribute. */
13187 2673960 : ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
13188 2665858 : || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
13189 : cost_vec)
13190 2665357 : || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
13191 2577806 : || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
13192 2154398 : || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
13193 2082483 : || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
13194 1617121 : || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
13195 780427 : || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
13196 731130 : || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
13197 703945 : || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
13198 556409 : || (bb_vinfo
13199 141182 : && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
13200 3169308 : || (is_a <loop_vec_info> (vinfo)
13201 415227 : && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
13202 : stmt_info, node, cost_vec)
13203 414487 : || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
13204 : stmt_info,
13205 : node, node_instance, cost_vec)
13206 334721 : || vectorizable_induction (as_a <loop_vec_info> (vinfo),
13207 : stmt_info, node, cost_vec)
13208 206990 : || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
13209 : stmt_info, node)
13210 206185 : || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13211 : stmt_info, node, cost_vec)
13212 205916 : || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13213 : stmt_info, NULL, node,
13214 : cost_vec))));
13215 :
13216 2673960 : STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
13217 :
13218 2387923 : if (!ok)
13219 286037 : return opt_result::failure_at (stmt_info->stmt,
13220 : "not vectorized:"
13221 : " relevant stmt not supported: %G",
13222 : stmt_info->stmt);
13223 :
13224 : /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13225 : need extra handling, except for vectorizable reductions. */
13226 2387923 : if (!bb_vinfo
13227 1339089 : && (SLP_TREE_TYPE (node) != lc_phi_info_type
13228 805 : || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
13229 1339089 : && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
13230 3727012 : && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
13231 : node, node_instance,
13232 : false, cost_vec))
13233 0 : return opt_result::failure_at (stmt_info->stmt,
13234 : "not vectorized:"
13235 : " live stmt not supported: %G",
13236 : stmt_info->stmt);
13237 :
13238 2387923 : return opt_result::success ();
13239 : }
13240 :
13241 :
13242 : /* Function vect_transform_stmt.
13243 :
13244 : Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13245 :
13246 : void
13247 976994 : vect_transform_stmt (vec_info *vinfo,
13248 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
13249 : slp_tree slp_node, slp_instance slp_node_instance)
13250 : {
13251 976994 : bool done;
13252 :
13253 976994 : gcc_assert (slp_node);
13254 :
13255 976994 : if (stmt_info)
13256 960086 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13257 :
13258 976994 : switch (SLP_TREE_TYPE (slp_node))
13259 : {
13260 23709 : case type_demotion_vec_info_type:
13261 23709 : case type_promotion_vec_info_type:
13262 23709 : case type_conversion_vec_info_type:
13263 23709 : done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
13264 23709 : gcc_assert (done);
13265 : break;
13266 :
13267 17011 : case induc_vec_info_type:
13268 17011 : done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
13269 : stmt_info, slp_node, NULL);
13270 17011 : gcc_assert (done);
13271 : break;
13272 :
13273 9232 : case shift_vec_info_type:
13274 9232 : done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
13275 9232 : gcc_assert (done);
13276 : break;
13277 :
13278 119230 : case op_vec_info_type:
13279 119230 : done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
13280 119230 : gcc_assert (done);
13281 : break;
13282 :
13283 16563 : case assignment_vec_info_type:
13284 16563 : done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
13285 16563 : gcc_assert (done);
13286 : break;
13287 :
13288 157666 : case load_vec_info_type:
13289 157666 : done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
13290 157666 : gcc_assert (done);
13291 : break;
13292 :
13293 550512 : case store_vec_info_type:
13294 550512 : done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
13295 550512 : gcc_assert (done);
13296 : break;
13297 :
13298 8739 : case condition_vec_info_type:
13299 8739 : done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
13300 8739 : gcc_assert (done);
13301 : break;
13302 :
13303 12817 : case comparison_vec_info_type:
13304 12817 : done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
13305 12817 : gcc_assert (done);
13306 : break;
13307 :
13308 4611 : case call_vec_info_type:
13309 4611 : done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
13310 4611 : break;
13311 :
13312 382 : case call_simd_clone_vec_info_type:
13313 382 : done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
13314 : slp_node, NULL);
13315 382 : break;
13316 :
13317 2692 : case reduc_vec_info_type:
13318 2692 : done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
13319 : gsi, slp_node);
13320 2692 : gcc_assert (done);
13321 : break;
13322 :
13323 23672 : case cycle_phi_info_type:
13324 23672 : done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
13325 : slp_node, slp_node_instance);
13326 23672 : gcc_assert (done);
13327 : break;
13328 :
13329 489 : case lc_phi_info_type:
13330 489 : done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
13331 : stmt_info, slp_node);
13332 489 : gcc_assert (done);
13333 : break;
13334 :
13335 43 : case recurr_info_type:
13336 43 : done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13337 : stmt_info, slp_node, NULL);
13338 43 : gcc_assert (done);
13339 : break;
13340 :
13341 12718 : case phi_info_type:
13342 12718 : done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
13343 : stmt_info, slp_node, NULL);
13344 12718 : gcc_assert (done);
13345 : break;
13346 :
13347 0 : case loop_exit_ctrl_vec_info_type:
13348 0 : done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13349 : stmt_info, gsi, slp_node, NULL);
13350 0 : gcc_assert (done);
13351 : break;
13352 :
13353 16908 : case permute_info_type:
13354 16908 : done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
13355 16908 : gcc_assert (done);
13356 : break;
13357 :
13358 0 : default:
13359 0 : if (!STMT_VINFO_LIVE_P (stmt_info))
13360 : {
13361 0 : if (dump_enabled_p ())
13362 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13363 : "stmt not supported.\n");
13364 0 : gcc_unreachable ();
13365 : }
13366 976994 : done = true;
13367 : }
13368 :
13369 976994 : if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
13370 426482 : && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
13371 : {
13372 : /* Handle stmts whose DEF is used outside the loop-nest that is
13373 : being vectorized. */
13374 536038 : for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
13375 : {
13376 40382 : stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
13377 40382 : done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13378 : slp_node_instance, lane,
13379 : true, NULL);
13380 40382 : gcc_assert (done);
13381 : }
13382 : }
13383 976994 : }
13384 :
13385 :
13386 : /* Remove a group of stores (for SLP or interleaving), free their
13387 : stmt_vec_info. */
13388 :
13389 : void
13390 0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
13391 : {
13392 0 : stmt_vec_info next_stmt_info = first_stmt_info;
13393 :
13394 0 : while (next_stmt_info)
13395 : {
13396 0 : stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
13397 0 : next_stmt_info = vect_orig_stmt (next_stmt_info);
13398 : /* Free the attached stmt_vec_info and remove the stmt. */
13399 0 : vinfo->remove_stmt (next_stmt_info);
13400 0 : next_stmt_info = tmp;
13401 : }
13402 0 : }
13403 :
13404 : /* If NUNITS is nonzero, return a vector type that contains NUNITS
13405 : elements of type SCALAR_TYPE, or null if the target doesn't support
13406 : such a type.
13407 :
13408 : If NUNITS is zero, return a vector type that contains elements of
13409 : type SCALAR_TYPE, choosing whichever vector size the target prefers.
13410 :
13411 : If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13412 : for this vectorization region and want to "autodetect" the best choice.
13413 : Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13414 : and we want the new type to be interoperable with it. PREVAILING_MODE
13415 : in this case can be a scalar integer mode or a vector mode; when it
13416 : is a vector mode, the function acts like a tree-level version of
13417 : related_vector_mode. */
13418 :
13419 : tree
13420 30945187 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
13421 : tree scalar_type, poly_uint64 nunits)
13422 : {
13423 30945187 : tree orig_scalar_type = scalar_type;
13424 30945187 : scalar_mode inner_mode;
13425 30945187 : machine_mode simd_mode;
13426 30945187 : tree vectype;
13427 :
13428 30945187 : if ((!INTEGRAL_TYPE_P (scalar_type)
13429 10177717 : && !POINTER_TYPE_P (scalar_type)
13430 2162166 : && !SCALAR_FLOAT_TYPE_P (scalar_type))
13431 40566816 : || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
13432 1606231 : && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
13433 : return NULL_TREE;
13434 :
13435 30385722 : unsigned int nbytes = GET_MODE_SIZE (inner_mode);
13436 :
13437 : /* Interoperability between modes requires one to be a constant multiple
13438 : of the other, so that the number of vectors required for each operation
13439 : is a compile-time constant. */
13440 30385722 : if (prevailing_mode != VOIDmode
13441 29215109 : && !constant_multiple_p (nunits * nbytes,
13442 29215109 : GET_MODE_SIZE (prevailing_mode))
13443 31630753 : && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
13444 1245031 : nunits * nbytes))
13445 : return NULL_TREE;
13446 :
13447 : /* For vector types of elements whose mode precision doesn't
13448 : match their types precision we use a element type of mode
13449 : precision. The vectorization routines will have to make sure
13450 : they support the proper result truncation/extension.
13451 : We also make sure to build vector types with INTEGER_TYPE
13452 : component type only. */
13453 30385722 : if (INTEGRAL_TYPE_P (scalar_type)
13454 51153039 : && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
13455 19546089 : || TREE_CODE (scalar_type) != INTEGER_TYPE))
13456 1437181 : scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
13457 1437181 : TYPE_UNSIGNED (scalar_type));
13458 :
13459 : /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13460 : When the component mode passes the above test simply use a type
13461 : corresponding to that mode. The theory is that any use that
13462 : would cause problems with this will disable vectorization anyway. */
13463 28948541 : else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
13464 : && !INTEGRAL_TYPE_P (scalar_type))
13465 8015551 : scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
13466 :
13467 : /* We can't build a vector type of elements with alignment bigger than
13468 : their size. */
13469 20932990 : else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
13470 421268 : scalar_type = lang_hooks.types.type_for_mode (inner_mode,
13471 210634 : TYPE_UNSIGNED (scalar_type));
13472 :
13473 : /* If we felt back to using the mode fail if there was
13474 : no scalar type for it. */
13475 30385722 : if (scalar_type == NULL_TREE)
13476 : return NULL_TREE;
13477 :
13478 : /* If no prevailing mode was supplied, use the mode the target prefers.
13479 : Otherwise lookup a vector mode based on the prevailing mode. */
13480 30385722 : if (prevailing_mode == VOIDmode)
13481 : {
13482 1170613 : gcc_assert (known_eq (nunits, 0U));
13483 1170613 : simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
13484 1170613 : if (SCALAR_INT_MODE_P (simd_mode))
13485 : {
13486 : /* Traditional behavior is not to take the integer mode
13487 : literally, but simply to use it as a way of determining
13488 : the vector size. It is up to mode_for_vector to decide
13489 : what the TYPE_MODE should be.
13490 :
13491 : Note that nunits == 1 is allowed in order to support single
13492 : element vector types. */
13493 66490 : if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
13494 33245 : || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13495 : return NULL_TREE;
13496 : }
13497 : }
13498 29215109 : else if (SCALAR_INT_MODE_P (prevailing_mode)
13499 29215109 : || !related_vector_mode (prevailing_mode,
13500 29214452 : inner_mode, nunits).exists (&simd_mode))
13501 : {
13502 : /* Fall back to using mode_for_vector, mostly in the hope of being
13503 : able to use an integer mode. */
13504 2386307 : if (known_eq (nunits, 0U)
13505 4767004 : && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
13506 : return NULL_TREE;
13507 :
13508 242143 : if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13509 226753 : return NULL_TREE;
13510 : }
13511 :
13512 27982261 : vectype = build_vector_type_for_mode (scalar_type, simd_mode);
13513 :
13514 : /* In cases where the mode was chosen by mode_for_vector, check that
13515 : the target actually supports the chosen mode, or that it at least
13516 : allows the vector mode to be replaced by a like-sized integer. */
13517 55964522 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
13518 27997960 : && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
13519 : return NULL_TREE;
13520 :
13521 : /* Re-attach the address-space qualifier if we canonicalized the scalar
13522 : type. */
13523 27974048 : if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
13524 3 : return build_qualified_type
13525 3 : (vectype, qualifier_set {TYPE_UNQUALIFIED,
13526 3 : TYPE_ADDR_SPACE (orig_scalar_type)});
13527 :
13528 : return vectype;
13529 : }
13530 :
13531 : /* Function get_vectype_for_scalar_type.
13532 :
13533 : Returns the vector type corresponding to SCALAR_TYPE as supported
13534 : by the target. If GROUP_SIZE is nonzero and we're performing BB
13535 : vectorization, make sure that the number of elements in the vector
13536 : is no bigger than GROUP_SIZE. */
13537 :
13538 : tree
13539 26987820 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
13540 : unsigned int group_size)
13541 : {
13542 : /* For BB vectorization, we should always have a group size once we've
13543 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13544 : are tentative requests during things like early data reference
13545 : analysis and pattern recognition. */
13546 26987820 : if (is_a <bb_vec_info> (vinfo))
13547 23804536 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
13548 : else
13549 : group_size = 0;
13550 :
13551 26987820 : tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13552 : scalar_type);
13553 26987820 : if (vectype && vinfo->vector_mode == VOIDmode)
13554 1085863 : vinfo->vector_mode = TYPE_MODE (vectype);
13555 :
13556 : /* Register the natural choice of vector type, before the group size
13557 : has been applied. */
13558 0 : if (vectype)
13559 24191415 : vinfo->used_vector_modes.add (TYPE_MODE (vectype));
13560 :
13561 : /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13562 : try again with an explicit number of elements. */
13563 24191415 : if (vectype
13564 24191415 : && group_size
13565 26987820 : && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
13566 : {
13567 : /* Start with the biggest number of units that fits within
13568 : GROUP_SIZE and halve it until we find a valid vector type.
13569 : Usually either the first attempt will succeed or all will
13570 : fail (in the latter case because GROUP_SIZE is too small
13571 : for the target), but it's possible that a target could have
13572 : a hole between supported vector types.
13573 :
13574 : If GROUP_SIZE is not a power of 2, this has the effect of
13575 : trying the largest power of 2 that fits within the group,
13576 : even though the group is not a multiple of that vector size.
13577 : The BB vectorizer will then try to carve up the group into
13578 : smaller pieces. */
13579 2590231 : unsigned int nunits = 1 << floor_log2 (group_size);
13580 2590231 : do
13581 : {
13582 2590231 : vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13583 2590231 : scalar_type, nunits);
13584 2590231 : nunits /= 2;
13585 : }
13586 2590231 : while (nunits > 1 && !vectype);
13587 : }
13588 :
13589 26987820 : return vectype;
13590 : }
13591 :
13592 : /* Return the vector type corresponding to SCALAR_TYPE as supported
13593 : by the target. NODE, if nonnull, is the SLP tree node that will
13594 : use the returned vector type. */
13595 :
13596 : tree
13597 184274 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
13598 : {
13599 184274 : unsigned int group_size = 0;
13600 184274 : if (node)
13601 184274 : group_size = SLP_TREE_LANES (node);
13602 184274 : return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13603 : }
13604 :
13605 : /* Function get_mask_type_for_scalar_type.
13606 :
13607 : Returns the mask type corresponding to a result of comparison
13608 : of vectors of specified SCALAR_TYPE as supported by target.
13609 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
13610 : make sure that the number of elements in the vector is no bigger
13611 : than GROUP_SIZE. */
13612 :
13613 : tree
13614 1158751 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13615 : unsigned int group_size)
13616 : {
13617 1158751 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13618 :
13619 1158751 : if (!vectype)
13620 : return NULL;
13621 :
13622 1136932 : return truth_type_for (vectype);
13623 : }
13624 :
13625 : /* Function get_mask_type_for_scalar_type.
13626 :
13627 : Returns the mask type corresponding to a result of comparison
13628 : of vectors of specified SCALAR_TYPE as supported by target.
13629 : NODE, if nonnull, is the SLP tree node that will use the returned
13630 : vector type. */
13631 :
13632 : tree
13633 19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13634 : slp_tree node)
13635 : {
13636 19 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
13637 :
13638 19 : if (!vectype)
13639 : return NULL;
13640 :
13641 19 : return truth_type_for (vectype);
13642 : }
13643 :
13644 : /* Function get_same_sized_vectype
13645 :
13646 : Returns a vector type corresponding to SCALAR_TYPE of size
13647 : VECTOR_TYPE if supported by the target. */
13648 :
13649 : tree
13650 1110 : get_same_sized_vectype (tree scalar_type, tree vector_type)
13651 : {
13652 1110 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
13653 0 : return truth_type_for (vector_type);
13654 :
13655 1110 : poly_uint64 nunits;
13656 2220 : if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
13657 2220 : GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
13658 : return NULL_TREE;
13659 :
13660 1110 : return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
13661 1110 : scalar_type, nunits);
13662 : }
13663 :
13664 : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13665 : would not change the chosen vector modes. */
13666 :
13667 : bool
13668 1495663 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
13669 : {
13670 1495663 : for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
13671 2580205 : i != vinfo->used_vector_modes.end (); ++i)
13672 1985655 : if (!VECTOR_MODE_P (*i)
13673 5956695 : || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
13674 901113 : return false;
13675 594550 : return true;
13676 : }
13677 :
13678 : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
13679 : change the chosen vector modes for analysis of a loop. */
13680 :
13681 : bool
13682 404081 : vect_chooses_same_modes_p (machine_mode vector_mode,
13683 : machine_mode alt_vector_mode)
13684 : {
13685 66091 : return (VECTOR_MODE_P (vector_mode)
13686 404081 : && VECTOR_MODE_P (alt_vector_mode)
13687 808162 : && (related_vector_mode (vector_mode,
13688 : GET_MODE_INNER (alt_vector_mode))
13689 404081 : == alt_vector_mode)
13690 431543 : && (related_vector_mode (alt_vector_mode,
13691 : GET_MODE_INNER (vector_mode))
13692 13731 : == vector_mode));
13693 : }
13694 :
13695 : /* Function vect_is_simple_use.
13696 :
13697 : Input:
13698 : VINFO - the vect info of the loop or basic block that is being vectorized.
13699 : OPERAND - operand in the loop or bb.
13700 : Output:
13701 : DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13702 : case OPERAND is an SSA_NAME that is defined in the vectorizable region
13703 : DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13704 : the definition could be anywhere in the function
13705 : DT - the type of definition
13706 :
13707 : Returns whether a stmt with OPERAND can be vectorized.
13708 : For loops, supportable operands are constants, loop invariants, and operands
13709 : that are defined by the current iteration of the loop. Unsupportable
13710 : operands are those that are defined by a previous iteration of the loop (as
13711 : is the case in reduction/induction computations).
13712 : For basic blocks, supportable operands are constants and bb invariants.
13713 : For now, operands defined outside the basic block are not supported. */
13714 :
13715 : bool
13716 44459959 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
13717 : stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
13718 : {
13719 44459959 : if (def_stmt_info_out)
13720 42410225 : *def_stmt_info_out = NULL;
13721 44459959 : if (def_stmt_out)
13722 10353972 : *def_stmt_out = NULL;
13723 44459959 : *dt = vect_unknown_def_type;
13724 :
13725 44459959 : if (dump_enabled_p ())
13726 : {
13727 818656 : dump_printf_loc (MSG_NOTE, vect_location,
13728 : "vect_is_simple_use: operand ");
13729 818656 : if (TREE_CODE (operand) == SSA_NAME
13730 818656 : && !SSA_NAME_IS_DEFAULT_DEF (operand))
13731 748091 : dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
13732 : else
13733 70565 : dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
13734 : }
13735 :
13736 44459959 : if (CONSTANT_CLASS_P (operand))
13737 3402669 : *dt = vect_constant_def;
13738 41057290 : else if (is_gimple_min_invariant (operand))
13739 400074 : *dt = vect_external_def;
13740 40657216 : else if (TREE_CODE (operand) != SSA_NAME)
13741 1064 : *dt = vect_unknown_def_type;
13742 40656152 : else if (SSA_NAME_IS_DEFAULT_DEF (operand))
13743 727021 : *dt = vect_external_def;
13744 : else
13745 : {
13746 39929131 : gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
13747 39929131 : stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
13748 39929131 : if (!stmt_vinfo)
13749 880101 : *dt = vect_external_def;
13750 : else
13751 : {
13752 39049030 : stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
13753 39049030 : def_stmt = stmt_vinfo->stmt;
13754 39049030 : *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
13755 39049030 : if (def_stmt_info_out)
13756 37008242 : *def_stmt_info_out = stmt_vinfo;
13757 : }
13758 39929131 : if (def_stmt_out)
13759 10124715 : *def_stmt_out = def_stmt;
13760 : }
13761 :
13762 44459959 : if (dump_enabled_p ())
13763 : {
13764 818656 : dump_printf (MSG_NOTE, ", type of def: ");
13765 818656 : switch (*dt)
13766 : {
13767 0 : case vect_uninitialized_def:
13768 0 : dump_printf (MSG_NOTE, "uninitialized\n");
13769 0 : break;
13770 58854 : case vect_constant_def:
13771 58854 : dump_printf (MSG_NOTE, "constant\n");
13772 58854 : break;
13773 28045 : case vect_external_def:
13774 28045 : dump_printf (MSG_NOTE, "external\n");
13775 28045 : break;
13776 583347 : case vect_internal_def:
13777 583347 : dump_printf (MSG_NOTE, "internal\n");
13778 583347 : break;
13779 115946 : case vect_induction_def:
13780 115946 : dump_printf (MSG_NOTE, "induction\n");
13781 115946 : break;
13782 29050 : case vect_reduction_def:
13783 29050 : dump_printf (MSG_NOTE, "reduction\n");
13784 29050 : break;
13785 482 : case vect_double_reduction_def:
13786 482 : dump_printf (MSG_NOTE, "double reduction\n");
13787 482 : break;
13788 2193 : case vect_nested_cycle:
13789 2193 : dump_printf (MSG_NOTE, "nested cycle\n");
13790 2193 : break;
13791 277 : case vect_first_order_recurrence:
13792 277 : dump_printf (MSG_NOTE, "first order recurrence\n");
13793 277 : break;
13794 0 : case vect_condition_def:
13795 0 : dump_printf (MSG_NOTE, "control flow\n");
13796 0 : break;
13797 462 : case vect_unknown_def_type:
13798 462 : dump_printf (MSG_NOTE, "unknown\n");
13799 462 : break;
13800 : }
13801 : }
13802 :
13803 44459959 : if (*dt == vect_unknown_def_type)
13804 : {
13805 60483 : if (dump_enabled_p ())
13806 462 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13807 : "Unsupported pattern.\n");
13808 : return false;
13809 : }
13810 :
13811 : return true;
13812 : }
13813 :
13814 : /* Function vect_is_simple_use.
13815 :
13816 : Same as vect_is_simple_use but determines the operand by operand
13817 : position OPERAND from either STMT or SLP_NODE, filling in *OP
13818 : and *SLP_DEF (when SLP_NODE is not NULL). */
13819 :
13820 : bool
13821 3729113 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13822 : unsigned operand, tree *op, slp_tree *slp_def,
13823 : enum vect_def_type *dt,
13824 : tree *vectype, stmt_vec_info *def_stmt_info_out)
13825 : {
13826 3729113 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
13827 3729113 : *slp_def = child;
13828 3729113 : *vectype = SLP_TREE_VECTYPE (child);
13829 3729113 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
13830 : {
13831 : /* ??? VEC_PERM nodes might be intermediate and their lane value
13832 : have no representative (nor do we build a VEC_PERM stmt for
13833 : the actual operation). Note for two-operator nodes we set
13834 : a representative but leave scalar stmts empty as we'd only
13835 : have one for a subset of lanes. Ideally no caller would
13836 : require *op for internal defs. */
13837 2025085 : if (SLP_TREE_REPRESENTATIVE (child))
13838 : {
13839 1967651 : *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
13840 1967651 : return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
13841 : }
13842 : else
13843 : {
13844 57434 : gcc_assert (SLP_TREE_PERMUTE_P (child));
13845 57434 : *op = error_mark_node;
13846 57434 : *dt = vect_internal_def;
13847 57434 : if (def_stmt_info_out)
13848 155 : *def_stmt_info_out = NULL;
13849 : return true;
13850 : }
13851 : }
13852 : else
13853 : {
13854 1704028 : if (def_stmt_info_out)
13855 55907 : *def_stmt_info_out = NULL;
13856 1704028 : *op = SLP_TREE_SCALAR_OPS (child)[0];
13857 1704028 : *dt = SLP_TREE_DEF_TYPE (child);
13858 1704028 : return true;
13859 : }
13860 : }
13861 :
13862 : /* Wrapper around vect_is_simple_use that elides the scalar op output. */
13863 :
13864 : bool
13865 285565 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13866 : unsigned operand, slp_tree *slp_def,
13867 : enum vect_def_type *dt, tree *vectype)
13868 : {
13869 285565 : tree op;
13870 285565 : return vect_is_simple_use (vinfo, slp_node, operand, &op, slp_def, dt,
13871 285565 : vectype);
13872 : }
13873 :
13874 : /* If OP is not NULL and is external or constant update its vector
13875 : type with VECTYPE. Returns true if successful or false if not,
13876 : for example when conflicting vector types are present. */
13877 :
13878 : bool
13879 3253881 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
13880 : {
13881 3253881 : if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
13882 : return true;
13883 1123942 : if (SLP_TREE_VECTYPE (op))
13884 23916 : return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
13885 : /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13886 : should be handled by patters. Allow vect_constant_def for now
13887 : as well as the trivial single-lane uniform vect_external_def case
13888 : both of which we code-generate reasonably. */
13889 1100026 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
13890 1713 : && SLP_TREE_DEF_TYPE (op) == vect_external_def
13891 1101272 : && SLP_TREE_LANES (op) > 1)
13892 : return false;
13893 1099836 : SLP_TREE_VECTYPE (op) = vectype;
13894 1099836 : return true;
13895 : }
13896 :
13897 : /* Function supportable_widening_operation
13898 :
13899 : Check whether an operation represented by the code CODE is a
13900 : widening operation that is supported by the target platform in
13901 : vector form (i.e., when operating on arguments of type VECTYPE_IN
13902 : producing a result of type VECTYPE_OUT).
13903 :
13904 : Widening operations we currently support are NOP (CONVERT), FLOAT,
13905 : FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13906 : are supported by the target platform either directly (via vector
13907 : tree-codes), or via target builtins.
13908 :
13909 : When EVENODD_OK then also lane-swizzling operations are considered.
13910 :
13911 : Output:
13912 : - CODE1 and CODE2 are codes of vector operations to be used when
13913 : vectorizing the operation, if available.
13914 : - MULTI_STEP_CVT determines the number of required intermediate steps in
13915 : case of multi-step conversion (like char->short->int - in that case
13916 : MULTI_STEP_CVT will be 1).
13917 : - INTERM_TYPES contains the intermediate type required to perform the
13918 : widening operation (short in the above example). */
13919 :
13920 : bool
13921 556888 : supportable_widening_operation (code_helper code,
13922 : tree vectype_out, tree vectype_in,
13923 : bool evenodd_ok,
13924 : code_helper *code1,
13925 : code_helper *code2,
13926 : int *multi_step_cvt,
13927 : vec<tree> *interm_types)
13928 : {
13929 556888 : machine_mode vec_mode;
13930 556888 : enum insn_code icode1, icode2;
13931 556888 : optab optab1 = unknown_optab, optab2 = unknown_optab;
13932 556888 : tree vectype = vectype_in;
13933 556888 : tree wide_vectype = vectype_out;
13934 556888 : tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
13935 556888 : int i;
13936 556888 : tree prev_type, intermediate_type;
13937 556888 : machine_mode intermediate_mode, prev_mode;
13938 556888 : optab optab3, optab4;
13939 :
13940 556888 : *multi_step_cvt = 0;
13941 :
13942 556888 : switch (code.safe_as_tree_code ())
13943 : {
13944 : case MAX_TREE_CODES:
13945 : /* Don't set c1 and c2 if code is not a tree_code. */
13946 : break;
13947 :
13948 211589 : case WIDEN_MULT_EXPR:
13949 : /* The result of a vectorized widening operation usually requires
13950 : two vectors (because the widened results do not fit into one vector).
13951 : The generated vector results would normally be expected to be
13952 : generated in the same order as in the original scalar computation,
13953 : i.e. if 8 results are generated in each vector iteration, they are
13954 : to be organized as follows:
13955 : vect1: [res1,res2,res3,res4],
13956 : vect2: [res5,res6,res7,res8].
13957 :
13958 : However, in the special case that the result of the widening
13959 : operation is used in a reduction computation only, the order doesn't
13960 : matter (because when vectorizing a reduction we change the order of
13961 : the computation). Some targets can take advantage of this and
13962 : generate more efficient code. For example, targets like Altivec,
13963 : that support widen_mult using a sequence of {mult_even,mult_odd}
13964 : generate the following vectors:
13965 : vect1: [res1,res3,res5,res7],
13966 : vect2: [res2,res4,res6,res8].
13967 :
13968 : When vectorizing outer-loops, we execute the inner-loop sequentially
13969 : (each vectorized inner-loop iteration contributes to VF outer-loop
13970 : iterations in parallel). We therefore don't allow to change the
13971 : order of the computation in the inner-loop during outer-loop
13972 : vectorization. */
13973 : /* TODO: Another case in which order doesn't *really* matter is when we
13974 : widen and then contract again, e.g. (short)((int)x * y >> 8).
13975 : Normally, pack_trunc performs an even/odd permute, whereas the
13976 : repack from an even/odd expansion would be an interleave, which
13977 : would be significantly simpler for e.g. AVX2. */
13978 : /* In any case, in order to avoid duplicating the code below, recurse
13979 : on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13980 : are properly set up for the caller. If we fail, we'll continue with
13981 : a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13982 211589 : if (evenodd_ok
13983 211589 : && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
13984 : vectype_out, vectype_in,
13985 : evenodd_ok, code1,
13986 : code2, multi_step_cvt,
13987 : interm_types))
13988 108456 : return true;
13989 : c1 = VEC_WIDEN_MULT_LO_EXPR;
13990 : c2 = VEC_WIDEN_MULT_HI_EXPR;
13991 : break;
13992 :
13993 : case DOT_PROD_EXPR:
13994 448432 : c1 = DOT_PROD_EXPR;
13995 448432 : c2 = DOT_PROD_EXPR;
13996 : break;
13997 :
13998 0 : case SAD_EXPR:
13999 0 : c1 = SAD_EXPR;
14000 0 : c2 = SAD_EXPR;
14001 0 : break;
14002 :
14003 209447 : case VEC_WIDEN_MULT_EVEN_EXPR:
14004 : /* Support the recursion induced just above. */
14005 209447 : c1 = VEC_WIDEN_MULT_EVEN_EXPR;
14006 209447 : c2 = VEC_WIDEN_MULT_ODD_EXPR;
14007 209447 : break;
14008 :
14009 9813 : case WIDEN_LSHIFT_EXPR:
14010 9813 : c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
14011 9813 : c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
14012 9813 : break;
14013 :
14014 45515 : CASE_CONVERT:
14015 45515 : c1 = VEC_UNPACK_LO_EXPR;
14016 45515 : c2 = VEC_UNPACK_HI_EXPR;
14017 45515 : break;
14018 :
14019 9281 : case FLOAT_EXPR:
14020 9281 : c1 = VEC_UNPACK_FLOAT_LO_EXPR;
14021 9281 : c2 = VEC_UNPACK_FLOAT_HI_EXPR;
14022 9281 : break;
14023 :
14024 127 : case FIX_TRUNC_EXPR:
14025 127 : c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
14026 127 : c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
14027 127 : break;
14028 :
14029 0 : default:
14030 0 : gcc_unreachable ();
14031 : }
14032 :
14033 448432 : if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
14034 : std::swap (c1, c2);
14035 :
14036 448432 : if (code == FIX_TRUNC_EXPR)
14037 : {
14038 : /* The signedness is determined from output operand. */
14039 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14040 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14041 : }
14042 788620 : else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
14043 45515 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14044 8257 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14045 8257 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14046 377653 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14047 : {
14048 : /* If the input and result modes are the same, a different optab
14049 : is needed where we pass in the number of units in vectype. */
14050 : optab1 = vec_unpacks_sbool_lo_optab;
14051 : optab2 = vec_unpacks_sbool_hi_optab;
14052 : }
14053 :
14054 448432 : vec_mode = TYPE_MODE (vectype);
14055 448432 : if (widening_fn_p (code))
14056 : {
14057 : /* If this is an internal fn then we must check whether the target
14058 : supports either a low-high split or an even-odd split. */
14059 71116 : internal_fn ifn = as_internal_fn ((combined_fn) code);
14060 :
14061 71116 : internal_fn lo, hi, even, odd;
14062 71116 : lookup_hilo_internal_fn (ifn, &lo, &hi);
14063 71116 : if (BYTES_BIG_ENDIAN)
14064 : std::swap (lo, hi);
14065 71116 : *code1 = as_combined_fn (lo);
14066 71116 : *code2 = as_combined_fn (hi);
14067 71116 : optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
14068 71116 : optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
14069 :
14070 : /* If we don't support low-high, then check for even-odd. */
14071 71116 : if (!optab1
14072 71116 : || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14073 0 : || !optab2
14074 71116 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14075 : {
14076 71116 : lookup_evenodd_internal_fn (ifn, &even, &odd);
14077 71116 : *code1 = as_combined_fn (even);
14078 71116 : *code2 = as_combined_fn (odd);
14079 71116 : optab1 = direct_internal_fn_optab (even, {vectype, vectype});
14080 71116 : optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
14081 : }
14082 : }
14083 377316 : else if (code.is_tree_code ())
14084 : {
14085 377316 : if (code == FIX_TRUNC_EXPR)
14086 : {
14087 : /* The signedness is determined from output operand. */
14088 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14089 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14090 : }
14091 377189 : else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
14092 45515 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14093 8257 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14094 8257 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14095 377653 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14096 : {
14097 : /* If the input and result modes are the same, a different optab
14098 : is needed where we pass in the number of units in vectype. */
14099 : optab1 = vec_unpacks_sbool_lo_optab;
14100 : optab2 = vec_unpacks_sbool_hi_optab;
14101 : }
14102 : else
14103 : {
14104 376725 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14105 376725 : optab2 = optab_for_tree_code (c2, vectype, optab_default);
14106 : }
14107 377316 : *code1 = c1;
14108 377316 : *code2 = c2;
14109 : }
14110 :
14111 448432 : if (!optab1 || !optab2)
14112 : return false;
14113 :
14114 448432 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14115 448432 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14116 : return false;
14117 :
14118 :
14119 168632 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14120 168632 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14121 : {
14122 156885 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14123 : return true;
14124 : /* For scalar masks we may have different boolean
14125 : vector types having the same QImode. Thus we
14126 : add additional check for elements number. */
14127 4459 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
14128 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14129 : return true;
14130 : }
14131 :
14132 : /* Check if it's a multi-step conversion that can be done using intermediate
14133 : types. */
14134 :
14135 11976 : prev_type = vectype;
14136 11976 : prev_mode = vec_mode;
14137 :
14138 11976 : if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
14139 : return false;
14140 :
14141 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14142 : intermediate steps in promotion sequence. We try
14143 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14144 : not. */
14145 11924 : interm_types->create (MAX_INTERM_CVT_STEPS);
14146 25170 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14147 : {
14148 13246 : intermediate_mode = insn_data[icode1].operand[0].mode;
14149 13246 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14150 4699 : intermediate_type
14151 4699 : = vect_halve_mask_nunits (prev_type, intermediate_mode);
14152 8547 : else if (VECTOR_MODE_P (intermediate_mode))
14153 : {
14154 8547 : tree intermediate_element_type
14155 8547 : = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
14156 8547 : TYPE_UNSIGNED (prev_type));
14157 8547 : intermediate_type
14158 8547 : = build_vector_type_for_mode (intermediate_element_type,
14159 : intermediate_mode);
14160 8547 : }
14161 : else
14162 0 : intermediate_type
14163 0 : = lang_hooks.types.type_for_mode (intermediate_mode,
14164 0 : TYPE_UNSIGNED (prev_type));
14165 :
14166 13246 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14167 4699 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14168 4699 : && intermediate_mode == TYPE_MODE (wide_vectype)
14169 13557 : && SCALAR_INT_MODE_P (intermediate_mode))
14170 : {
14171 : /* If the input and result modes are the same, a different optab
14172 : is needed where we pass in the number of units in vectype. */
14173 : optab3 = vec_unpacks_sbool_lo_optab;
14174 : optab4 = vec_unpacks_sbool_hi_optab;
14175 : }
14176 : else
14177 : {
14178 12935 : optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
14179 12935 : optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
14180 : }
14181 :
14182 13246 : if (!optab3 || !optab4
14183 13246 : || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
14184 13206 : || insn_data[icode1].operand[0].mode != intermediate_mode
14185 13206 : || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
14186 13206 : || insn_data[icode2].operand[0].mode != intermediate_mode
14187 13206 : || ((icode1 = optab_handler (optab3, intermediate_mode))
14188 : == CODE_FOR_nothing)
14189 26068 : || ((icode2 = optab_handler (optab4, intermediate_mode))
14190 : == CODE_FOR_nothing))
14191 : break;
14192 :
14193 12822 : interm_types->quick_push (intermediate_type);
14194 12822 : (*multi_step_cvt)++;
14195 :
14196 12822 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14197 12822 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14198 : {
14199 11582 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14200 : return true;
14201 3774 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
14202 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14203 : return true;
14204 : }
14205 :
14206 1322 : prev_type = intermediate_type;
14207 1322 : prev_mode = intermediate_mode;
14208 : }
14209 :
14210 424 : interm_types->release ();
14211 424 : return false;
14212 : }
14213 :
14214 :
14215 : /* Function supportable_narrowing_operation
14216 :
14217 : Check whether an operation represented by the code CODE is a
14218 : narrowing operation that is supported by the target platform in
14219 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14220 : and producing a result of type VECTYPE_OUT).
14221 :
14222 : Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14223 : and FLOAT. This function checks if these operations are supported by
14224 : the target platform directly via vector tree-codes.
14225 :
14226 : Output:
14227 : - CODE1 is the code of a vector operation to be used when
14228 : vectorizing the operation, if available.
14229 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14230 : case of multi-step conversion (like int->short->char - in that case
14231 : MULTI_STEP_CVT will be 1).
14232 : - INTERM_TYPES contains the intermediate type required to perform the
14233 : narrowing operation (short in the above example). */
14234 :
14235 : bool
14236 45351 : supportable_narrowing_operation (code_helper code,
14237 : tree vectype_out, tree vectype_in,
14238 : code_helper *code1, int *multi_step_cvt,
14239 : vec<tree> *interm_types)
14240 : {
14241 45351 : machine_mode vec_mode;
14242 45351 : enum insn_code icode1;
14243 45351 : optab optab1, interm_optab;
14244 45351 : tree vectype = vectype_in;
14245 45351 : tree narrow_vectype = vectype_out;
14246 45351 : enum tree_code c1;
14247 45351 : tree intermediate_type, prev_type;
14248 45351 : machine_mode intermediate_mode, prev_mode;
14249 45351 : int i;
14250 45351 : unsigned HOST_WIDE_INT n_elts;
14251 45351 : bool uns;
14252 :
14253 45351 : if (!code.is_tree_code ())
14254 : return false;
14255 :
14256 45351 : *multi_step_cvt = 0;
14257 45351 : switch ((tree_code) code)
14258 : {
14259 43841 : CASE_CONVERT:
14260 43841 : c1 = VEC_PACK_TRUNC_EXPR;
14261 43841 : if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
14262 11630 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14263 11630 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
14264 5271 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
14265 49112 : && n_elts < BITS_PER_UNIT)
14266 : optab1 = vec_pack_sbool_trunc_optab;
14267 : else
14268 41354 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14269 : break;
14270 :
14271 572 : case FIX_TRUNC_EXPR:
14272 572 : c1 = VEC_PACK_FIX_TRUNC_EXPR;
14273 : /* The signedness is determined from output operand. */
14274 572 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14275 572 : break;
14276 :
14277 938 : case FLOAT_EXPR:
14278 938 : c1 = VEC_PACK_FLOAT_EXPR;
14279 938 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14280 938 : break;
14281 :
14282 0 : default:
14283 0 : gcc_unreachable ();
14284 : }
14285 :
14286 45351 : if (!optab1)
14287 : return false;
14288 :
14289 45351 : vec_mode = TYPE_MODE (vectype);
14290 45351 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
14291 : return false;
14292 :
14293 39787 : *code1 = c1;
14294 :
14295 39787 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14296 : {
14297 25126 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14298 : return true;
14299 : /* For scalar masks we may have different boolean
14300 : vector types having the same QImode. Thus we
14301 : add additional check for elements number. */
14302 5833 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
14303 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14304 : return true;
14305 : }
14306 :
14307 14812 : if (code == FLOAT_EXPR)
14308 : return false;
14309 :
14310 : /* Check if it's a multi-step conversion that can be done using intermediate
14311 : types. */
14312 14812 : prev_mode = vec_mode;
14313 14812 : prev_type = vectype;
14314 14812 : if (code == FIX_TRUNC_EXPR)
14315 94 : uns = TYPE_UNSIGNED (vectype_out);
14316 : else
14317 14718 : uns = TYPE_UNSIGNED (vectype);
14318 :
14319 : /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14320 : conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14321 : costly than signed. */
14322 14812 : if (code == FIX_TRUNC_EXPR && uns)
14323 : {
14324 28 : enum insn_code icode2;
14325 :
14326 28 : intermediate_type
14327 28 : = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
14328 28 : interm_optab
14329 28 : = optab_for_tree_code (c1, intermediate_type, optab_default);
14330 28 : if (interm_optab != unknown_optab
14331 28 : && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
14332 28 : && insn_data[icode1].operand[0].mode
14333 28 : == insn_data[icode2].operand[0].mode)
14334 : {
14335 : uns = false;
14336 : optab1 = interm_optab;
14337 : icode1 = icode2;
14338 : }
14339 : }
14340 :
14341 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14342 : intermediate steps in promotion sequence. We try
14343 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14344 14812 : interm_types->create (MAX_INTERM_CVT_STEPS);
14345 31708 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14346 : {
14347 16896 : intermediate_mode = insn_data[icode1].operand[0].mode;
14348 16896 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14349 7028 : intermediate_type
14350 7028 : = vect_double_mask_nunits (prev_type, intermediate_mode);
14351 : else
14352 9868 : intermediate_type
14353 9868 : = lang_hooks.types.type_for_mode (intermediate_mode, uns);
14354 16896 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14355 7028 : && VECTOR_BOOLEAN_TYPE_P (prev_type)
14356 7028 : && SCALAR_INT_MODE_P (prev_mode)
14357 3144 : && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
14358 20040 : && n_elts < BITS_PER_UNIT)
14359 : interm_optab = vec_pack_sbool_trunc_optab;
14360 : else
14361 16544 : interm_optab
14362 16544 : = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
14363 : optab_default);
14364 352 : if (!interm_optab
14365 16896 : || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
14366 16896 : || insn_data[icode1].operand[0].mode != intermediate_mode
14367 33440 : || ((icode1 = optab_handler (interm_optab, intermediate_mode))
14368 : == CODE_FOR_nothing))
14369 : break;
14370 :
14371 15965 : interm_types->quick_push (intermediate_type);
14372 15965 : (*multi_step_cvt)++;
14373 :
14374 15965 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14375 : {
14376 13881 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14377 : return true;
14378 4962 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
14379 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14380 : return true;
14381 : }
14382 :
14383 2084 : prev_mode = intermediate_mode;
14384 2084 : prev_type = intermediate_type;
14385 2084 : optab1 = interm_optab;
14386 : }
14387 :
14388 931 : interm_types->release ();
14389 931 : return false;
14390 : }
14391 :
14392 : /* Function supportable_indirect_convert_operation
14393 :
14394 : Check whether an operation represented by the code CODE is single or multi
14395 : operations that are supported by the target platform in
14396 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14397 : producing a result of type VECTYPE_OUT).
14398 :
14399 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14400 : This function checks if these operations are supported
14401 : by the target platform directly (via vector tree-codes).
14402 :
14403 : Output:
14404 : - converts contains some pairs to perform the convert operation,
14405 : the pair's first is the intermediate type, and its second is the code of
14406 : a vector operation to be used when converting the operation from the
14407 : previous type to the intermediate type. */
14408 : bool
14409 94811 : supportable_indirect_convert_operation (code_helper code,
14410 : tree vectype_out,
14411 : tree vectype_in,
14412 : vec<std::pair<tree, tree_code> > &converts,
14413 : slp_tree slp_op0)
14414 : {
14415 94811 : bool found_mode = false;
14416 94811 : scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
14417 94811 : scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
14418 94811 : tree_code code1, code2;
14419 :
14420 94811 : tree cvt_type = NULL_TREE;
14421 94811 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
14422 :
14423 94811 : if (supportable_convert_operation ((tree_code) code,
14424 : vectype_out,
14425 : vectype_in))
14426 : {
14427 87240 : converts.safe_push (std::make_pair (vectype_out, (tree_code) code));
14428 87240 : return true;
14429 : }
14430 :
14431 : /* For conversions between mask types where the destination has
14432 : a data mode attempt a vcond_mask conversion. */
14433 7571 : if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
14434 145 : && VECTOR_BOOLEAN_TYPE_P (vectype_out)
14435 7716 : && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
14436 : {
14437 144 : tree scalar_datatype
14438 144 : = build_nonstandard_integer_type (element_precision (vectype_out), 0);
14439 144 : tree datatype_out = build_vector_type_for_mode (scalar_datatype,
14440 144 : TYPE_MODE (vectype_out));
14441 144 : if (expand_vec_cond_expr_p (datatype_out, vectype_in))
14442 : {
14443 2 : converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
14444 2 : return true;
14445 : }
14446 : }
14447 :
14448 : /* For conversions between float and integer types try whether
14449 : we can use intermediate signed integer types to support the
14450 : conversion. */
14451 15138 : if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
14452 7569 : && (code == FLOAT_EXPR
14453 4933 : || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
14454 : {
14455 1592 : bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
14456 796 : bool float_expr_p = code == FLOAT_EXPR;
14457 796 : unsigned short target_size;
14458 796 : scalar_mode intermediate_mode;
14459 796 : if (demotion)
14460 : {
14461 99 : intermediate_mode = lhs_mode;
14462 99 : target_size = GET_MODE_SIZE (rhs_mode);
14463 : }
14464 : else
14465 : {
14466 697 : target_size = GET_MODE_SIZE (lhs_mode);
14467 697 : if (!int_mode_for_size
14468 697 : (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
14469 142 : return false;
14470 : }
14471 796 : code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
14472 729 : code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
14473 796 : opt_scalar_mode mode_iter;
14474 2483 : FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
14475 : {
14476 2483 : intermediate_mode = mode_iter.require ();
14477 :
14478 4966 : if (GET_MODE_SIZE (intermediate_mode) > target_size)
14479 : break;
14480 :
14481 1882 : scalar_mode cvt_mode;
14482 1882 : if (!int_mode_for_size
14483 1882 : (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
14484 : break;
14485 :
14486 1852 : cvt_type = build_nonstandard_integer_type
14487 1852 : (GET_MODE_BITSIZE (cvt_mode), 0);
14488 :
14489 : /* Check if the intermediate type can hold OP0's range.
14490 : When converting from float to integer this is not necessary
14491 : because values that do not fit the (smaller) target type are
14492 : unspecified anyway. */
14493 1852 : if (demotion && float_expr_p)
14494 : {
14495 23 : wide_int op_min_value, op_max_value;
14496 : /* For vector form, it looks like slp_op0 doesn't have RANGE_INFO.
14497 : In the future, if it is supported, changes may need to be made
14498 : to this part, such as checking the RANGE of each element
14499 : in the vector. */
14500 23 : if (slp_op0)
14501 : {
14502 13 : tree def;
14503 : /* ??? Merge ranges in case of more than one lane. */
14504 13 : if (SLP_TREE_LANES (slp_op0) != 1
14505 0 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
14506 13 : || !vect_get_range_info (def,
14507 : &op_min_value, &op_max_value))
14508 : break;
14509 : }
14510 : else
14511 : break;
14512 :
14513 0 : if (cvt_type == NULL_TREE
14514 0 : || (wi::min_precision (op_max_value, SIGNED)
14515 0 : > TYPE_PRECISION (cvt_type))
14516 0 : || (wi::min_precision (op_min_value, SIGNED)
14517 0 : > TYPE_PRECISION (cvt_type)))
14518 0 : continue;
14519 23 : }
14520 :
14521 1829 : cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
14522 : cvt_type,
14523 : nelts);
14524 : /* This should only happened for SLP as long as loop vectorizer
14525 : only supports same-sized vector. */
14526 3516 : if (cvt_type == NULL_TREE
14527 3658 : || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
14528 1829 : || !supportable_convert_operation ((tree_code) code1,
14529 : vectype_out, cvt_type)
14530 2564 : || !supportable_convert_operation ((tree_code) code2,
14531 : cvt_type, vectype_in))
14532 1687 : continue;
14533 :
14534 : found_mode = true;
14535 : break;
14536 : }
14537 :
14538 796 : if (found_mode)
14539 : {
14540 142 : converts.safe_push (std::make_pair (cvt_type, code2));
14541 142 : if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
14542 142 : converts.safe_push (std::make_pair (vectype_out, code1));
14543 : return true;
14544 : }
14545 : }
14546 : return false;
14547 : }
14548 :
14549 : /* Generate and return a vector mask of MASK_TYPE such that
14550 : mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14551 : Add the statements to SEQ. */
14552 :
14553 : tree
14554 0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
14555 : tree end_index, const char *name)
14556 : {
14557 0 : tree cmp_type = TREE_TYPE (start_index);
14558 0 : gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
14559 : cmp_type, mask_type,
14560 : OPTIMIZE_FOR_SPEED));
14561 0 : gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
14562 : start_index, end_index,
14563 : build_zero_cst (mask_type));
14564 0 : tree tmp;
14565 0 : if (name)
14566 0 : tmp = make_temp_ssa_name (mask_type, NULL, name);
14567 : else
14568 0 : tmp = make_ssa_name (mask_type);
14569 0 : gimple_call_set_lhs (call, tmp);
14570 0 : gimple_seq_add_stmt (seq, call);
14571 0 : return tmp;
14572 : }
14573 :
14574 : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14575 : J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14576 :
14577 : tree
14578 0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
14579 : tree end_index)
14580 : {
14581 0 : tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
14582 0 : return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
14583 : }
14584 :
14585 : /* Try to compute the vector types required to vectorize STMT_INFO,
14586 : returning true on success and false if vectorization isn't possible.
14587 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
14588 : take sure that the number of elements in the vectors is no bigger
14589 : than GROUP_SIZE.
14590 :
14591 : On success:
14592 :
14593 : - Set *STMT_VECTYPE_OUT to:
14594 : - NULL_TREE if the statement doesn't need to be vectorized;
14595 : - the equivalent of STMT_VINFO_VECTYPE otherwise. */
14596 :
14597 : opt_result
14598 6252099 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
14599 : tree *stmt_vectype_out,
14600 : unsigned int group_size)
14601 : {
14602 6252099 : gimple *stmt = stmt_info->stmt;
14603 :
14604 : /* For BB vectorization, we should always have a group size once we've
14605 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14606 : are tentative requests during things like early data reference
14607 : analysis and pattern recognition. */
14608 6252099 : if (is_a <bb_vec_info> (vinfo))
14609 4967593 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
14610 : else
14611 : group_size = 0;
14612 :
14613 6252099 : *stmt_vectype_out = NULL_TREE;
14614 :
14615 6252099 : if (gimple_get_lhs (stmt) == NULL_TREE
14616 : /* Allow vector conditionals through here. */
14617 2657 : && !is_a <gcond *> (stmt)
14618 : /* MASK_STORE and friends have no lhs, but are ok. */
14619 6257383 : && !(is_gimple_call (stmt)
14620 2657 : && gimple_call_internal_p (stmt)
14621 2627 : && internal_store_fn_p (gimple_call_internal_fn (stmt))))
14622 : {
14623 30 : if (is_a <gcall *> (stmt))
14624 : {
14625 : /* Ignore calls with no lhs. These must be calls to
14626 : #pragma omp simd functions, and what vectorization factor
14627 : it really needs can't be determined until
14628 : vectorizable_simd_clone_call. */
14629 30 : if (dump_enabled_p ())
14630 28 : dump_printf_loc (MSG_NOTE, vect_location,
14631 : "defer to SIMD clone analysis.\n");
14632 30 : return opt_result::success ();
14633 : }
14634 :
14635 0 : return opt_result::failure_at (stmt,
14636 : "not vectorized: irregular stmt: %G", stmt);
14637 : }
14638 :
14639 6252069 : tree vectype;
14640 6252069 : tree scalar_type = NULL_TREE;
14641 6252069 : if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
14642 : {
14643 1708990 : vectype = STMT_VINFO_VECTYPE (stmt_info);
14644 1708990 : if (dump_enabled_p ())
14645 85433 : dump_printf_loc (MSG_NOTE, vect_location,
14646 : "precomputed vectype: %T\n", vectype);
14647 : }
14648 4543079 : else if (vect_use_mask_type_p (stmt_info))
14649 : {
14650 229022 : unsigned int precision = stmt_info->mask_precision;
14651 229022 : scalar_type = build_nonstandard_integer_type (precision, 1);
14652 229022 : vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
14653 229022 : if (!vectype)
14654 0 : return opt_result::failure_at (stmt, "not vectorized: unsupported"
14655 : " data-type %T\n", scalar_type);
14656 229022 : if (dump_enabled_p ())
14657 5102 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14658 : }
14659 : else
14660 : {
14661 : /* If we got here with a gcond it means that the target had no available vector
14662 : mode for the scalar type. We can't vectorize so abort. */
14663 4314057 : if (is_a <gcond *> (stmt))
14664 0 : return opt_result::failure_at (stmt,
14665 : "not vectorized:"
14666 : " unsupported data-type for gcond %T\n",
14667 : scalar_type);
14668 :
14669 4314057 : if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
14670 1532760 : scalar_type = TREE_TYPE (DR_REF (dr));
14671 : else
14672 2781297 : scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
14673 :
14674 4314057 : if (dump_enabled_p ())
14675 : {
14676 69626 : if (group_size)
14677 9298 : dump_printf_loc (MSG_NOTE, vect_location,
14678 : "get vectype for scalar type (group size %d):"
14679 : " %T\n", group_size, scalar_type);
14680 : else
14681 60328 : dump_printf_loc (MSG_NOTE, vect_location,
14682 : "get vectype for scalar type: %T\n", scalar_type);
14683 : }
14684 4314057 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
14685 4314057 : if (!vectype)
14686 257784 : return opt_result::failure_at (stmt,
14687 : "not vectorized:"
14688 : " unsupported data-type %T\n",
14689 : scalar_type);
14690 :
14691 4056273 : if (dump_enabled_p ())
14692 69355 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14693 : }
14694 :
14695 4370728 : if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
14696 0 : return opt_result::failure_at (stmt,
14697 : "not vectorized: vector stmt in loop:%G",
14698 : stmt);
14699 :
14700 5994285 : *stmt_vectype_out = vectype;
14701 5994285 : return opt_result::success ();
14702 : }
14703 :
14704 : /* Generate and return statement sequence that sets vector length LEN that is:
14705 :
14706 : min_of_start_and_end = min (START_INDEX, END_INDEX);
14707 : left_len = END_INDEX - min_of_start_and_end;
14708 : rhs = min (left_len, LEN_LIMIT);
14709 : LEN = rhs;
14710 :
14711 : Note: the cost of the code generated by this function is modeled
14712 : by vect_estimate_min_profitable_iters, so changes here may need
14713 : corresponding changes there. */
14714 :
14715 : gimple_seq
14716 0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
14717 : {
14718 0 : gimple_seq stmts = NULL;
14719 0 : tree len_type = TREE_TYPE (len);
14720 0 : gcc_assert (TREE_TYPE (start_index) == len_type);
14721 :
14722 0 : tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
14723 0 : tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
14724 0 : tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
14725 0 : gimple* stmt = gimple_build_assign (len, rhs);
14726 0 : gimple_seq_add_stmt (&stmts, stmt);
14727 :
14728 0 : return stmts;
14729 : }
14730 :
|