Line data Source code
1 : /* Statement Analysis and Transformation for Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "ssa.h"
31 : #include "optabs-tree.h"
32 : #include "insn-config.h"
33 : #include "recog.h" /* FIXME: for insn_data */
34 : #include "cgraph.h"
35 : #include "dumpfile.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "tree-eh.h"
40 : #include "gimplify.h"
41 : #include "gimple-iterator.h"
42 : #include "gimplify-me.h"
43 : #include "tree-cfg.h"
44 : #include "tree-ssa-loop-manip.h"
45 : #include "cfgloop.h"
46 : #include "explow.h"
47 : #include "tree-ssa-loop.h"
48 : #include "tree-scalar-evolution.h"
49 : #include "tree-vectorizer.h"
50 : #include "builtins.h"
51 : #include "internal-fn.h"
52 : #include "tree-vector-builder.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "tree-ssa-loop-niter.h"
56 : #include "gimple-fold.h"
57 : #include "regs.h"
58 : #include "attribs.h"
59 : #include "optabs-libfuncs.h"
60 : #include "tree-dfa.h"
61 :
62 : /* For lang_hooks.types.type_for_mode. */
63 : #include "langhooks.h"
64 :
65 : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
66 : bool = false);
67 :
68 : /* Return TRUE iff the given statement is in an inner loop relative to
69 : the loop being vectorized. */
70 : bool
71 5928087 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
72 : {
73 5928087 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
74 5928087 : basic_block bb = gimple_bb (stmt);
75 5928087 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
76 2790485 : class loop* loop;
77 :
78 2790485 : if (!loop_vinfo)
79 : return false;
80 :
81 2790485 : loop = LOOP_VINFO_LOOP (loop_vinfo);
82 :
83 2790485 : return (bb->loop_father == loop->inner);
84 : }
85 :
86 : /* Record the cost of a statement, either by directly informing the
87 : target model or by saving it in a vector for later processing.
88 : Return a preliminary estimate of the statement's cost. */
89 :
90 : unsigned
91 8979416 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
92 : enum vect_cost_for_stmt kind,
93 : stmt_vec_info stmt_info, slp_tree node,
94 : tree vectype, int misalign,
95 : enum vect_cost_model_location where)
96 : {
97 8979416 : if ((kind == vector_load || kind == unaligned_load)
98 1619433 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
99 : kind = vector_gather_load;
100 8979416 : if ((kind == vector_store || kind == unaligned_store)
101 1032595 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
102 8979416 : kind = vector_scatter_store;
103 :
104 8979416 : stmt_info_for_cost si
105 8979416 : = { count, kind, where, stmt_info, node, vectype, misalign };
106 8979416 : body_cost_vec->safe_push (si);
107 :
108 8979416 : return (unsigned)
109 8979416 : (builtin_vectorization_cost (kind, vectype, misalign) * count);
110 : }
111 :
112 : unsigned
113 4043692 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
114 : enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
115 : tree vectype, int misalign,
116 : enum vect_cost_model_location where)
117 : {
118 4043692 : return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
119 4043692 : vectype, misalign, where);
120 : }
121 :
122 : unsigned
123 1808767 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
124 : enum vect_cost_for_stmt kind, slp_tree node,
125 : tree vectype, int misalign,
126 : enum vect_cost_model_location where)
127 : {
128 1808767 : return record_stmt_cost (body_cost_vec, count, kind,
129 : SLP_TREE_REPRESENTATIVE (node), node,
130 1808767 : vectype, misalign, where);
131 : }
132 :
133 : unsigned
134 0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
135 : enum vect_cost_for_stmt kind,
136 : enum vect_cost_model_location where)
137 : {
138 0 : gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
139 : || kind == scalar_stmt);
140 0 : return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
141 0 : NULL_TREE, 0, where);
142 : }
143 :
144 : /* Return a variable of type ELEM_TYPE[NELEMS]. */
145 :
146 : static tree
147 0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
148 : {
149 0 : return create_tmp_var (build_array_type_nelts (elem_type, nelems),
150 0 : "vect_array");
151 : }
152 :
153 : /* ARRAY is an array of vectors created by create_vector_array.
154 : Return an SSA_NAME for the vector in index N. The reference
155 : is part of the vectorization of STMT_INFO and the vector is associated
156 : with scalar destination SCALAR_DEST.
157 : If we need to ensure that inactive elements are set to zero,
158 : NEED_ZEROING is true, MASK contains the loop mask to be used. */
159 :
160 : static tree
161 0 : read_vector_array (vec_info *vinfo,
162 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
163 : tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
164 : bool need_zeroing, tree mask)
165 : {
166 0 : tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
167 0 : gimple *new_stmt;
168 :
169 0 : gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
170 0 : vect_type = TREE_TYPE (TREE_TYPE (array));
171 0 : tmp = vect_create_destination_var (scalar_dest, vect_type);
172 0 : vect = vect_create_destination_var (scalar_dest, vect_type);
173 0 : array_ref = build4 (ARRAY_REF, vect_type, array,
174 0 : build_int_cst (size_type_node, n),
175 : NULL_TREE, NULL_TREE);
176 :
177 0 : new_stmt = gimple_build_assign (tmp, array_ref);
178 0 : tmp_name = make_ssa_name (vect, new_stmt);
179 0 : gimple_assign_set_lhs (new_stmt, tmp_name);
180 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
181 :
182 0 : if (need_zeroing)
183 : {
184 0 : tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
185 : vect_type);
186 0 : vect_name = make_ssa_name (vect, new_stmt);
187 0 : new_stmt
188 0 : = gimple_build_assign (vect_name, VEC_COND_EXPR,
189 : mask, tmp_name, vec_els);
190 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
191 : }
192 : else
193 : vect_name = tmp_name;
194 :
195 0 : return vect_name;
196 : }
197 :
198 : /* ARRAY is an array of vectors created by create_vector_array.
199 : Emit code to store SSA_NAME VECT in index N of the array.
200 : The store is part of the vectorization of STMT_INFO. */
201 :
202 : static void
203 0 : write_vector_array (vec_info *vinfo,
204 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
205 : tree vect, tree array, unsigned HOST_WIDE_INT n)
206 : {
207 0 : tree array_ref;
208 0 : gimple *new_stmt;
209 :
210 0 : array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
211 0 : build_int_cst (size_type_node, n),
212 : NULL_TREE, NULL_TREE);
213 :
214 0 : new_stmt = gimple_build_assign (array_ref, vect);
215 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
216 0 : }
217 :
218 : /* PTR is a pointer to an array of type TYPE. Return a representation
219 : of *PTR. The memory reference replaces those in FIRST_DR
220 : (and its group). */
221 :
222 : static tree
223 0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
224 : {
225 0 : tree mem_ref;
226 :
227 0 : mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
228 : /* Arrays have the same alignment as their type. */
229 0 : set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
230 0 : return mem_ref;
231 : }
232 :
233 : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
234 : Emit the clobber before *GSI. */
235 :
236 : static void
237 15 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
238 : gimple_stmt_iterator *gsi, tree var)
239 : {
240 15 : tree clobber = build_clobber (TREE_TYPE (var));
241 15 : gimple *new_stmt = gimple_build_assign (var, clobber);
242 15 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
243 15 : }
244 :
245 : /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
246 :
247 : /* Function vect_mark_relevant.
248 :
249 : Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
250 :
251 : static void
252 3250159 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
253 : enum vect_relevant relevant, bool live_p)
254 : {
255 3250159 : enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
256 3250159 : bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
257 :
258 3250159 : if (dump_enabled_p ())
259 164263 : dump_printf_loc (MSG_NOTE, vect_location,
260 : "mark relevant %d, live %d: %G", relevant, live_p,
261 : stmt_info->stmt);
262 :
263 : /* If this stmt is an original stmt in a pattern, we might need to mark its
264 : related pattern stmt instead of the original stmt. However, such stmts
265 : may have their own uses that are not in any pattern, in such cases the
266 : stmt itself should be marked. */
267 3250159 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
268 : {
269 : /* This is the last stmt in a sequence that was detected as a
270 : pattern that can potentially be vectorized. Don't mark the stmt
271 : as relevant/live because it's not going to be vectorized.
272 : Instead mark the pattern-stmt that replaces it. */
273 :
274 249318 : if (dump_enabled_p ())
275 2815 : dump_printf_loc (MSG_NOTE, vect_location,
276 : "last stmt in pattern. don't mark"
277 : " relevant/live.\n");
278 :
279 249318 : stmt_vec_info old_stmt_info = stmt_info;
280 249318 : stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
281 249318 : gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
282 249318 : save_relevant = STMT_VINFO_RELEVANT (stmt_info);
283 249318 : save_live_p = STMT_VINFO_LIVE_P (stmt_info);
284 :
285 249318 : if (live_p && relevant == vect_unused_in_scope)
286 : {
287 110 : if (dump_enabled_p ())
288 10 : dump_printf_loc (MSG_NOTE, vect_location,
289 : "vec_stmt_relevant_p: forcing live pattern stmt "
290 : "relevant.\n");
291 : relevant = vect_used_only_live;
292 : }
293 :
294 249318 : if (dump_enabled_p ())
295 2815 : dump_printf_loc (MSG_NOTE, vect_location,
296 : "mark relevant %d, live %d: %G", relevant, live_p,
297 : stmt_info->stmt);
298 : }
299 :
300 3250159 : STMT_VINFO_LIVE_P (stmt_info) |= live_p;
301 3250159 : if (relevant > STMT_VINFO_RELEVANT (stmt_info))
302 2918011 : STMT_VINFO_RELEVANT (stmt_info) = relevant;
303 :
304 3250159 : if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
305 332148 : && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
306 : {
307 331435 : if (dump_enabled_p ())
308 19583 : dump_printf_loc (MSG_NOTE, vect_location,
309 : "already marked relevant/live.\n");
310 331435 : return;
311 : }
312 :
313 2918724 : worklist->safe_push (stmt_info);
314 : }
315 :
316 :
317 : /* Function is_simple_and_all_uses_invariant
318 :
319 : Return true if STMT_INFO is simple and all uses of it are invariant. */
320 :
321 : bool
322 251545 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
323 : loop_vec_info loop_vinfo)
324 : {
325 251545 : tree op;
326 251545 : ssa_op_iter iter;
327 :
328 447938 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
329 197216 : if (!stmt)
330 : return false;
331 :
332 204777 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
333 : {
334 203954 : enum vect_def_type dt = vect_uninitialized_def;
335 :
336 203954 : if (!vect_is_simple_use (op, loop_vinfo, &dt))
337 : {
338 5371 : if (dump_enabled_p ())
339 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
340 : "use not simple.\n");
341 196393 : return false;
342 : }
343 :
344 198583 : if (dt != vect_external_def && dt != vect_constant_def)
345 : return false;
346 : }
347 : return true;
348 : }
349 :
350 : /* Function vect_stmt_relevant_p.
351 :
352 : Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
353 : is "relevant for vectorization".
354 :
355 : A stmt is considered "relevant for vectorization" if:
356 : - it has uses outside the loop.
357 : - it has vdefs (it alters memory).
358 : - control stmts in the loop (except for the exit condition).
359 :
360 : CHECKME: what other side effects would the vectorizer allow? */
361 :
362 : static bool
363 5246825 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
364 : enum vect_relevant *relevant, bool *live_p)
365 : {
366 5246825 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
367 5246825 : ssa_op_iter op_iter;
368 5246825 : imm_use_iterator imm_iter;
369 5246825 : use_operand_p use_p;
370 5246825 : def_operand_p def_p;
371 :
372 5246825 : *relevant = vect_unused_in_scope;
373 5246825 : *live_p = false;
374 :
375 : /* cond stmt other than loop exit cond. */
376 5246825 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
377 5246825 : if (is_ctrl_stmt (stmt)
378 620298 : && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
379 5482805 : && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
380 233995 : *relevant = vect_used_in_scope;
381 :
382 : /* changing memory. */
383 5246825 : if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
384 4346461 : if (gimple_vdef (stmt_info->stmt)
385 3726163 : && !gimple_clobber_p (stmt_info->stmt))
386 : {
387 373799 : if (dump_enabled_p ())
388 28023 : dump_printf_loc (MSG_NOTE, vect_location,
389 : "vec_stmt_relevant_p: stmt has vdefs.\n");
390 373799 : *relevant = vect_used_in_scope;
391 373799 : if (! STMT_VINFO_DATA_REF (stmt_info)
392 373799 : && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
393 20 : LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
394 : }
395 :
396 : /* uses outside the loop. */
397 14739798 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
398 : {
399 15613692 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
400 : {
401 7121396 : basic_block bb = gimple_bb (USE_STMT (use_p));
402 7121396 : if (!flow_bb_inside_loop_p (loop, bb))
403 : {
404 266588 : if (is_gimple_debug (USE_STMT (use_p)))
405 1131 : continue;
406 :
407 265457 : if (dump_enabled_p ())
408 5991 : dump_printf_loc (MSG_NOTE, vect_location,
409 : "vec_stmt_relevant_p: used out of loop.\n");
410 :
411 : /* We expect all such uses to be in the loop exit phis
412 : (because of loop closed form) */
413 265457 : gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
414 :
415 265457 : *live_p = true;
416 265457 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
417 : }
418 4246148 : }
419 : }
420 :
421 251547 : if (*live_p && *relevant == vect_unused_in_scope
422 5498370 : && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
423 : {
424 250722 : if (dump_enabled_p ())
425 5847 : dump_printf_loc (MSG_NOTE, vect_location,
426 : "vec_stmt_relevant_p: stmt live but not relevant.\n");
427 250722 : *relevant = vect_used_only_live;
428 : }
429 :
430 5246825 : return (*live_p || *relevant);
431 : }
432 :
433 :
434 : /* Function exist_non_indexing_operands_for_use_p
435 :
436 : USE is one of the uses attached to STMT_INFO. Check if USE is
437 : used in STMT_INFO for anything other than indexing an array. */
438 :
439 : static bool
440 4356833 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
441 : {
442 4356833 : tree operand;
443 :
444 : /* USE corresponds to some operand in STMT. If there is no data
445 : reference in STMT, then any operand that corresponds to USE
446 : is not indexing an array. */
447 4356833 : if (!STMT_VINFO_DATA_REF (stmt_info))
448 : return true;
449 :
450 : /* STMT has a data_ref. FORNOW this means that its of one of
451 : the following forms:
452 : -1- ARRAY_REF = var
453 : -2- var = ARRAY_REF
454 : (This should have been verified in analyze_data_refs).
455 :
456 : 'var' in the second case corresponds to a def, not a use,
457 : so USE cannot correspond to any operands that are not used
458 : for array indexing.
459 :
460 : Therefore, all we need to check is if STMT falls into the
461 : first case, and whether var corresponds to USE. */
462 :
463 1486125 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
464 1467991 : if (!assign || !gimple_assign_copy_p (assign))
465 : {
466 796995 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
467 18134 : if (call && gimple_call_internal_p (call))
468 : {
469 18134 : internal_fn ifn = gimple_call_internal_fn (call);
470 18134 : int mask_index = internal_fn_mask_index (ifn);
471 18134 : if (mask_index >= 0
472 18134 : && use == gimple_call_arg (call, mask_index))
473 : return true;
474 11769 : int els_index = internal_fn_else_index (ifn);
475 11769 : if (els_index >= 0
476 11769 : && use == gimple_call_arg (call, els_index))
477 : return true;
478 10264 : int stored_value_index = internal_fn_stored_value_index (ifn);
479 10264 : if (stored_value_index >= 0
480 10264 : && use == gimple_call_arg (call, stored_value_index))
481 : return true;
482 8066 : if (internal_gather_scatter_fn_p (ifn)
483 8066 : && use == gimple_call_arg (call, 1))
484 : return true;
485 : }
486 786927 : return false;
487 : }
488 :
489 689130 : if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
490 : return false;
491 689130 : operand = gimple_assign_rhs1 (assign);
492 689130 : if (TREE_CODE (operand) != SSA_NAME)
493 : return false;
494 :
495 595870 : if (operand == use)
496 : return true;
497 :
498 : return false;
499 : }
500 :
501 :
502 : /*
503 : Function process_use.
504 :
505 : Inputs:
506 : - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
507 : - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
508 : that defined USE. This is done by calling mark_relevant and passing it
509 : the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
510 : - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
511 : be performed.
512 :
513 : Outputs:
514 : Generally, LIVE_P and RELEVANT are used to define the liveness and
515 : relevance info of the DEF_STMT of this USE:
516 : STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
517 : STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
518 : Exceptions:
519 : - case 1: If USE is used only for address computations (e.g. array indexing),
520 : which does not need to be directly vectorized, then the liveness/relevance
521 : of the respective DEF_STMT is left unchanged.
522 : - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
523 : we skip DEF_STMT cause it had already been processed.
524 : - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
525 : "relevant" will be modified accordingly.
526 :
527 : Return true if everything is as expected. Return false otherwise. */
528 :
529 : static opt_result
530 4413671 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
531 : enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
532 : bool force)
533 : {
534 4413671 : stmt_vec_info dstmt_vinfo;
535 4413671 : enum vect_def_type dt;
536 :
537 : /* case 1: we are only interested in uses that need to be vectorized. Uses
538 : that are used for address computation are not considered relevant. */
539 4413671 : if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
540 1194634 : return opt_result::success ();
541 :
542 3219037 : if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
543 35205 : return opt_result::failure_at (stmt_vinfo->stmt,
544 : "not vectorized:"
545 : " unsupported use in stmt.\n");
546 :
547 3183832 : if (!dstmt_vinfo)
548 601725 : return opt_result::success ();
549 :
550 2582107 : basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
551 2582107 : basic_block bb = gimple_bb (stmt_vinfo->stmt);
552 :
553 : /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
554 : We have to force the stmt live since the epilogue loop needs it to
555 : continue computing the reduction. */
556 2582107 : if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
557 272227 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
558 84957 : && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
559 84957 : && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
560 2667064 : && bb->loop_father == def_bb->loop_father)
561 : {
562 84957 : if (dump_enabled_p ())
563 3930 : dump_printf_loc (MSG_NOTE, vect_location,
564 : "reduc-stmt defining reduc-phi in the same nest.\n");
565 84957 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
566 84957 : return opt_result::success ();
567 : }
568 :
569 : /* case 3a: outer-loop stmt defining an inner-loop stmt:
570 : outer-loop-header-bb:
571 : d = dstmt_vinfo
572 : inner-loop:
573 : stmt # use (d)
574 : outer-loop-tail-bb:
575 : ... */
576 2497150 : if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
577 : {
578 2237 : if (dump_enabled_p ())
579 321 : dump_printf_loc (MSG_NOTE, vect_location,
580 : "outer-loop def-stmt defining inner-loop stmt.\n");
581 :
582 2237 : switch (relevant)
583 : {
584 0 : case vect_unused_in_scope:
585 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
586 : vect_used_in_scope : vect_unused_in_scope;
587 : break;
588 :
589 776 : case vect_used_in_outer_by_reduction:
590 776 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
591 : relevant = vect_used_by_reduction;
592 : break;
593 :
594 1181 : case vect_used_in_outer:
595 1181 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
596 : relevant = vect_used_in_scope;
597 : break;
598 :
599 : case vect_used_in_scope:
600 : break;
601 :
602 0 : default:
603 0 : gcc_unreachable ();
604 : }
605 : }
606 :
607 : /* case 3b: inner-loop stmt defining an outer-loop stmt:
608 : outer-loop-header-bb:
609 : ...
610 : inner-loop:
611 : d = dstmt_vinfo
612 : outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
613 : stmt # use (d) */
614 2494913 : else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
615 : {
616 2100 : if (dump_enabled_p ())
617 626 : dump_printf_loc (MSG_NOTE, vect_location,
618 : "inner-loop def-stmt defining outer-loop stmt.\n");
619 :
620 2100 : switch (relevant)
621 : {
622 0 : case vect_unused_in_scope:
623 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
624 0 : || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
625 : vect_used_in_outer_by_reduction : vect_unused_in_scope;
626 : break;
627 :
628 : case vect_used_by_reduction:
629 : case vect_used_only_live:
630 : relevant = vect_used_in_outer_by_reduction;
631 : break;
632 :
633 : case vect_used_in_scope:
634 2316257 : relevant = vect_used_in_outer;
635 : break;
636 :
637 0 : default:
638 0 : gcc_unreachable ();
639 : }
640 : }
641 : /* We are also not interested in uses on loop PHI backedges that are
642 : inductions. Otherwise we'll needlessly vectorize the IV increment
643 : and cause hybrid SLP for SLP inductions. */
644 2492813 : else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
645 183936 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
646 2673706 : && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
647 : loop_latch_edge (bb->loop_father))
648 : == use))
649 : {
650 180893 : if (dump_enabled_p ())
651 4864 : dump_printf_loc (MSG_NOTE, vect_location,
652 : "induction value on backedge.\n");
653 180893 : return opt_result::success ();
654 : }
655 :
656 2316257 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
657 2316257 : return opt_result::success ();
658 : }
659 :
660 :
661 : /* Function vect_mark_stmts_to_be_vectorized.
662 :
663 : Not all stmts in the loop need to be vectorized. For example:
664 :
665 : for i...
666 : for j...
667 : 1. T0 = i + j
668 : 2. T1 = a[T0]
669 :
670 : 3. j = j + 1
671 :
672 : Stmt 1 and 3 do not need to be vectorized, because loop control and
673 : addressing of vectorized data-refs are handled differently.
674 :
675 : This pass detects such stmts. */
676 :
677 : opt_result
678 441225 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
679 : {
680 441225 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
681 441225 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
682 441225 : unsigned int nbbs = loop->num_nodes;
683 441225 : gimple_stmt_iterator si;
684 441225 : unsigned int i;
685 441225 : basic_block bb;
686 441225 : bool live_p;
687 441225 : enum vect_relevant relevant;
688 :
689 441225 : DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
690 :
691 441225 : auto_vec<stmt_vec_info, 64> worklist;
692 :
693 : /* 1. Init worklist. */
694 1496204 : for (i = 0; i < nbbs; i++)
695 : {
696 1065517 : bb = bbs[i];
697 2188306 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
698 : {
699 2266366 : if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
700 232819 : continue;
701 900364 : stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
702 900364 : if (dump_enabled_p ())
703 41685 : dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
704 : phi_info->stmt);
705 :
706 900364 : if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
707 : {
708 45453 : if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
709 10394 : return opt_result::failure_at
710 10394 : (*si, "not vectorized: unhandled relevant PHI: %G", *si);
711 35059 : vect_mark_relevant (&worklist, phi_info, relevant, live_p);
712 : }
713 : }
714 8425080 : for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
715 : {
716 7370101 : gimple *stmt = gsi_stmt (si);
717 7370101 : if (is_gimple_debug (stmt))
718 3023496 : continue;
719 4346605 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
720 4346605 : if (dump_enabled_p ())
721 223340 : dump_printf_loc (MSG_NOTE, vect_location,
722 : "init: stmt relevant? %G", stmt);
723 :
724 4346605 : if (gimple_get_lhs (stmt) == NULL_TREE
725 626573 : && !is_a <gcond *> (stmt)
726 4352880 : && !is_a <gcall *> (stmt))
727 144 : return opt_result::failure_at
728 144 : (stmt, "not vectorized: irregular stmt: %G", stmt);
729 :
730 4346461 : if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
731 813886 : vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
732 : }
733 : }
734 :
735 : /* 2. Process_worklist */
736 3224520 : while (worklist.length () > 0)
737 : {
738 2829040 : use_operand_p use_p;
739 2829040 : ssa_op_iter iter;
740 :
741 2829040 : stmt_vec_info stmt_vinfo = worklist.pop ();
742 2829040 : if (dump_enabled_p ())
743 144040 : dump_printf_loc (MSG_NOTE, vect_location,
744 : "worklist: examine stmt: %G", stmt_vinfo->stmt);
745 :
746 : /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
747 : (DEF_STMT) as relevant/irrelevant according to the relevance property
748 : of STMT. */
749 2829040 : relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
750 :
751 : /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
752 : propagated as is to the DEF_STMTs of its USEs.
753 :
754 : One exception is when STMT has been identified as defining a reduction
755 : variable; in this case we set the relevance to vect_used_by_reduction.
756 : This is because we distinguish between two kinds of relevant stmts -
757 : those that are used by a reduction computation, and those that are
758 : (also) used by a regular computation. This allows us later on to
759 : identify stmts that are used solely by a reduction, and therefore the
760 : order of the results that they produce does not have to be kept. */
761 :
762 2829040 : switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
763 : {
764 172017 : case vect_reduction_def:
765 172017 : gcc_assert (relevant != vect_unused_in_scope);
766 172017 : if (relevant != vect_unused_in_scope
767 172017 : && relevant != vect_used_in_scope
768 172017 : && relevant != vect_used_by_reduction
769 172017 : && relevant != vect_used_only_live)
770 0 : return opt_result::failure_at
771 0 : (stmt_vinfo->stmt, "unsupported use of reduction.\n");
772 : break;
773 :
774 2209 : case vect_nested_cycle:
775 2209 : if (relevant != vect_unused_in_scope
776 2209 : && relevant != vect_used_in_outer_by_reduction
777 1614 : && relevant != vect_used_in_outer)
778 2 : return opt_result::failure_at
779 2 : (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
780 : break;
781 :
782 1209 : case vect_double_reduction_def:
783 1209 : if (relevant != vect_unused_in_scope
784 1209 : && relevant != vect_used_by_reduction
785 409 : && relevant != vect_used_only_live)
786 0 : return opt_result::failure_at
787 0 : (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
788 : break;
789 :
790 : default:
791 : break;
792 : }
793 :
794 2829038 : if (is_pattern_stmt_p (stmt_vinfo))
795 : {
796 : /* Pattern statements are not inserted into the code, so
797 : FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
798 : have to scan the RHS or function arguments instead. */
799 638774 : if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
800 : {
801 414315 : enum tree_code rhs_code = gimple_assign_rhs_code (assign);
802 414315 : tree op = gimple_assign_rhs1 (assign);
803 :
804 414315 : i = 1;
805 414315 : if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
806 : {
807 0 : opt_result res
808 0 : = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
809 : loop_vinfo, relevant, &worklist, false);
810 0 : if (!res)
811 0 : return res;
812 0 : res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
813 : loop_vinfo, relevant, &worklist, false);
814 0 : if (!res)
815 0 : return res;
816 : i = 2;
817 : }
818 1193301 : for (; i < gimple_num_ops (assign); i++)
819 : {
820 782838 : op = gimple_op (assign, i);
821 782838 : if (TREE_CODE (op) == SSA_NAME)
822 : {
823 594462 : opt_result res
824 594462 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
825 : &worklist, false);
826 594462 : if (!res)
827 3852 : return res;
828 : }
829 : }
830 : }
831 224459 : else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
832 : {
833 217832 : tree_code rhs_code = gimple_cond_code (cond);
834 217832 : gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
835 217832 : opt_result res
836 217832 : = process_use (stmt_vinfo, gimple_cond_lhs (cond),
837 : loop_vinfo, relevant, &worklist, false);
838 217832 : if (!res)
839 35207 : return res;
840 217832 : res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
841 : loop_vinfo, relevant, &worklist, false);
842 217832 : if (!res)
843 0 : return res;
844 : }
845 6627 : else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
846 : {
847 31687 : for (i = 0; i < gimple_call_num_args (call); i++)
848 : {
849 25060 : tree arg = gimple_call_arg (call, i);
850 25060 : opt_result res
851 25060 : = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
852 : &worklist, false);
853 25060 : if (!res)
854 0 : return res;
855 : }
856 : }
857 : else
858 0 : gcc_unreachable ();
859 : }
860 : else
861 7664107 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
862 : {
863 3301647 : tree op = USE_FROM_PTR (use_p);
864 3301647 : opt_result res
865 3301647 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
866 : &worklist, false);
867 3301647 : if (!res)
868 18068 : return res;
869 : }
870 :
871 2807118 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
872 : {
873 56838 : gather_scatter_info gs_info;
874 56838 : if (!vect_check_gather_scatter (stmt_vinfo,
875 : STMT_VINFO_VECTYPE (stmt_vinfo),
876 : loop_vinfo, &gs_info))
877 0 : gcc_unreachable ();
878 56838 : opt_result res
879 56838 : = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
880 : &worklist, true);
881 56838 : if (!res)
882 : {
883 13285 : if (fatal)
884 13285 : *fatal = false;
885 13285 : return res;
886 : }
887 : }
888 : } /* while worklist */
889 :
890 395480 : return opt_result::success ();
891 441225 : }
892 :
893 : /* Function vect_model_simple_cost.
894 :
895 : Models cost for simple operations, i.e. those that only emit N operations
896 : of the same KIND. */
897 :
898 : static void
899 787636 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
900 : stmt_vector_for_cost *cost_vec,
901 : vect_cost_for_stmt kind = vector_stmt)
902 : {
903 787636 : int inside_cost = 0, prologue_cost = 0;
904 :
905 787636 : gcc_assert (cost_vec != NULL);
906 :
907 787636 : n *= vect_get_num_copies (vinfo, node);
908 :
909 : /* Pass the inside-of-loop statements to the target-specific cost model. */
910 787636 : inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
911 :
912 787636 : if (dump_enabled_p ())
913 33269 : dump_printf_loc (MSG_NOTE, vect_location,
914 : "vect_model_simple_cost: inside_cost = %d, "
915 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
916 787636 : }
917 :
918 :
919 : /* Model cost for type demotion and promotion operations. PWR is
920 : normally zero for single-step promotions and demotions. It will be
921 : one if two-step promotion/demotion is required, and so on. NCOPIES
922 : is the number of vector results (and thus number of instructions)
923 : for the narrowest end of the operation chain. Each additional
924 : step doubles the number of instructions required. If WIDEN_ARITH
925 : is true the stmt is doing widening arithmetic. */
926 :
927 : static void
928 68670 : vect_model_promotion_demotion_cost (slp_tree slp_node,
929 : unsigned int ncopies, int pwr,
930 : stmt_vector_for_cost *cost_vec,
931 : bool widen_arith)
932 : {
933 68670 : int i;
934 68670 : int inside_cost = 0, prologue_cost = 0;
935 :
936 160000 : for (i = 0; i < pwr + 1; i++)
937 : {
938 180914 : inside_cost += record_stmt_cost (cost_vec, ncopies,
939 : widen_arith
940 : ? vector_stmt : vec_promote_demote,
941 : slp_node, 0, vect_body);
942 91330 : ncopies *= 2;
943 : }
944 :
945 68670 : if (dump_enabled_p ())
946 6368 : dump_printf_loc (MSG_NOTE, vect_location,
947 : "vect_model_promotion_demotion_cost: inside_cost = %d, "
948 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
949 68670 : }
950 :
951 : /* Returns true if the current function returns DECL. */
952 :
953 : static bool
954 566042 : cfun_returns (tree decl)
955 : {
956 566042 : edge_iterator ei;
957 566042 : edge e;
958 1114591 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
959 : {
960 1120730 : greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
961 560365 : if (!ret)
962 0 : continue;
963 560365 : if (gimple_return_retval (ret) == decl)
964 : return true;
965 : /* We often end up with an aggregate copy to the result decl,
966 : handle that case as well. First skip intermediate clobbers
967 : though. */
968 : gimple *def = ret;
969 1694220 : do
970 : {
971 3388440 : def = SSA_NAME_DEF_STMT (gimple_vuse (def));
972 : }
973 1694220 : while (gimple_clobber_p (def));
974 549346 : if (is_a <gassign *> (def)
975 62396 : && gimple_assign_lhs (def) == gimple_return_retval (ret)
976 556521 : && gimple_assign_rhs1 (def) == decl)
977 : return true;
978 : }
979 : return false;
980 : }
981 :
982 : /* Calculate cost of DR's memory access. */
983 : void
984 1022508 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
985 : int ncopies, dr_alignment_support alignment_support_scheme,
986 : int misalignment,
987 : unsigned int *inside_cost,
988 : stmt_vector_for_cost *body_cost_vec)
989 : {
990 1022508 : tree vectype
991 1022508 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
992 1022508 : switch (alignment_support_scheme)
993 : {
994 560362 : case dr_aligned:
995 560362 : {
996 560362 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
997 : vector_store, stmt_info, slp_node,
998 : vectype, 0, vect_body);
999 :
1000 560362 : if (dump_enabled_p ())
1001 14529 : dump_printf_loc (MSG_NOTE, vect_location,
1002 : "vect_model_store_cost: aligned.\n");
1003 : break;
1004 : }
1005 :
1006 462146 : case dr_unaligned_supported:
1007 462146 : {
1008 : /* Here, we assign an additional cost for the unaligned store. */
1009 462146 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1010 : unaligned_store, stmt_info, slp_node,
1011 : vectype, misalignment, vect_body);
1012 462146 : if (dump_enabled_p ())
1013 12916 : dump_printf_loc (MSG_NOTE, vect_location,
1014 : "vect_model_store_cost: unaligned supported by "
1015 : "hardware.\n");
1016 : break;
1017 : }
1018 :
1019 0 : case dr_unaligned_unsupported:
1020 0 : {
1021 0 : *inside_cost = VECT_MAX_COST;
1022 :
1023 0 : if (dump_enabled_p ())
1024 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1025 : "vect_model_store_cost: unsupported access.\n");
1026 : break;
1027 : }
1028 :
1029 0 : default:
1030 0 : gcc_unreachable ();
1031 : }
1032 1022508 : }
1033 :
1034 : /* Calculate cost of DR's memory access. */
1035 : void
1036 952576 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
1037 : int ncopies, dr_alignment_support alignment_support_scheme,
1038 : int misalignment,
1039 : bool add_realign_cost, unsigned int *inside_cost,
1040 : unsigned int *prologue_cost,
1041 : stmt_vector_for_cost *prologue_cost_vec,
1042 : stmt_vector_for_cost *body_cost_vec,
1043 : bool record_prologue_costs)
1044 : {
1045 952576 : tree vectype
1046 952576 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
1047 952576 : switch (alignment_support_scheme)
1048 : {
1049 546957 : case dr_aligned:
1050 546957 : {
1051 546957 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1052 : stmt_info, slp_node, vectype,
1053 : 0, vect_body);
1054 :
1055 546957 : if (dump_enabled_p ())
1056 18887 : dump_printf_loc (MSG_NOTE, vect_location,
1057 : "vect_model_load_cost: aligned.\n");
1058 :
1059 : break;
1060 : }
1061 345512 : case dr_unaligned_supported:
1062 345512 : {
1063 : /* Here, we assign an additional cost for the unaligned load. */
1064 345512 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1065 : unaligned_load, stmt_info, slp_node,
1066 : vectype, misalignment, vect_body);
1067 :
1068 345512 : if (dump_enabled_p ())
1069 22329 : dump_printf_loc (MSG_NOTE, vect_location,
1070 : "vect_model_load_cost: unaligned supported by "
1071 : "hardware.\n");
1072 :
1073 : break;
1074 : }
1075 0 : case dr_explicit_realign:
1076 0 : {
1077 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1078 : vector_load, stmt_info, slp_node,
1079 : vectype, 0, vect_body);
1080 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1081 : vec_perm, stmt_info, slp_node,
1082 : vectype, 0, vect_body);
1083 :
1084 : /* FIXME: If the misalignment remains fixed across the iterations of
1085 : the containing loop, the following cost should be added to the
1086 : prologue costs. */
1087 0 : if (targetm.vectorize.builtin_mask_for_load)
1088 0 : *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1089 : stmt_info, slp_node, vectype,
1090 : 0, vect_body);
1091 :
1092 0 : if (dump_enabled_p ())
1093 0 : dump_printf_loc (MSG_NOTE, vect_location,
1094 : "vect_model_load_cost: explicit realign\n");
1095 :
1096 : break;
1097 : }
1098 0 : case dr_explicit_realign_optimized:
1099 0 : {
1100 0 : if (dump_enabled_p ())
1101 0 : dump_printf_loc (MSG_NOTE, vect_location,
1102 : "vect_model_load_cost: unaligned software "
1103 : "pipelined.\n");
1104 :
1105 : /* Unaligned software pipeline has a load of an address, an initial
1106 : load, and possibly a mask operation to "prime" the loop. However,
1107 : if this is an access in a group of loads, which provide grouped
1108 : access, then the above cost should only be considered for one
1109 : access in the group. Inside the loop, there is a load op
1110 : and a realignment op. */
1111 :
1112 0 : if (add_realign_cost && record_prologue_costs)
1113 : {
1114 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1115 : vector_stmt, stmt_info,
1116 : slp_node, vectype,
1117 : 0, vect_prologue);
1118 0 : if (targetm.vectorize.builtin_mask_for_load)
1119 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1120 : vector_stmt, stmt_info,
1121 : slp_node, vectype,
1122 : 0, vect_prologue);
1123 : }
1124 :
1125 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1126 : stmt_info, slp_node, vectype,
1127 : 0, vect_body);
1128 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1129 : stmt_info, slp_node, vectype,
1130 : 0, vect_body);
1131 :
1132 0 : if (dump_enabled_p ())
1133 0 : dump_printf_loc (MSG_NOTE, vect_location,
1134 : "vect_model_load_cost: explicit realign optimized"
1135 : "\n");
1136 :
1137 : break;
1138 : }
1139 :
1140 60107 : case dr_unaligned_unsupported:
1141 60107 : {
1142 60107 : *inside_cost = VECT_MAX_COST;
1143 :
1144 60107 : if (dump_enabled_p ())
1145 104 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1146 : "vect_model_load_cost: unsupported access.\n");
1147 : break;
1148 : }
1149 :
1150 0 : default:
1151 0 : gcc_unreachable ();
1152 : }
1153 952576 : }
1154 :
1155 : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1156 : the loop preheader for the vectorized stmt STMT_VINFO. */
1157 :
1158 : static void
1159 6635 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
1160 : gimple_stmt_iterator *gsi)
1161 : {
1162 6635 : if (gsi)
1163 3344 : vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
1164 : else
1165 3291 : vinfo->insert_on_entry (stmt_vinfo, new_stmt);
1166 :
1167 6635 : if (dump_enabled_p ())
1168 1815 : dump_printf_loc (MSG_NOTE, vect_location,
1169 : "created new init_stmt: %G", new_stmt);
1170 6635 : }
1171 :
1172 : /* Function vect_init_vector.
1173 :
1174 : Insert a new stmt (INIT_STMT) that initializes a new variable of type
1175 : TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1176 : vector type a vector with all elements equal to VAL is created first.
1177 : Place the initialization at GSI if it is not NULL. Otherwise, place the
1178 : initialization at the loop preheader.
1179 : Return the DEF of INIT_STMT.
1180 : It will be used in the vectorization of STMT_INFO. */
1181 :
1182 : tree
1183 4920 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
1184 : gimple_stmt_iterator *gsi)
1185 : {
1186 4920 : gimple *init_stmt;
1187 4920 : tree new_temp;
1188 :
1189 : /* We abuse this function to push sth to a SSA name with initial 'val'. */
1190 4920 : if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1191 : {
1192 1331 : gcc_assert (VECTOR_TYPE_P (type));
1193 1331 : if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1194 : {
1195 : /* Scalar boolean value should be transformed into
1196 : all zeros or all ones value before building a vector. */
1197 11 : if (VECTOR_BOOLEAN_TYPE_P (type))
1198 : {
1199 3 : tree true_val = build_all_ones_cst (TREE_TYPE (type));
1200 3 : tree false_val = build_zero_cst (TREE_TYPE (type));
1201 :
1202 3 : if (CONSTANT_CLASS_P (val))
1203 0 : val = integer_zerop (val) ? false_val : true_val;
1204 : else
1205 : {
1206 3 : new_temp = make_ssa_name (TREE_TYPE (type));
1207 3 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1208 : val, true_val, false_val);
1209 3 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1210 3 : val = new_temp;
1211 : }
1212 : }
1213 : else
1214 : {
1215 8 : gimple_seq stmts = NULL;
1216 8 : if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1217 8 : val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
1218 8 : TREE_TYPE (type), val);
1219 : else
1220 : /* ??? Condition vectorization expects us to do
1221 : promotion of invariant/external defs. */
1222 0 : val = gimple_convert (&stmts, TREE_TYPE (type), val);
1223 16 : for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
1224 16 : !gsi_end_p (gsi2); )
1225 : {
1226 8 : init_stmt = gsi_stmt (gsi2);
1227 8 : gsi_remove (&gsi2, false);
1228 8 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1229 : }
1230 : }
1231 : }
1232 1331 : val = build_vector_from_val (type, val);
1233 : }
1234 :
1235 4920 : new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1236 4920 : init_stmt = gimple_build_assign (new_temp, val);
1237 4920 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1238 4920 : return new_temp;
1239 : }
1240 :
1241 :
1242 : /* Get vectorized definitions for OP0 and OP1. */
1243 :
1244 : void
1245 188362 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
1246 : tree op0, vec<tree> *vec_oprnds0,
1247 : tree op1, vec<tree> *vec_oprnds1,
1248 : tree op2, vec<tree> *vec_oprnds2,
1249 : tree op3, vec<tree> *vec_oprnds3)
1250 : {
1251 188362 : if (op0)
1252 186700 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
1253 188362 : if (op1)
1254 138501 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
1255 188362 : if (op2)
1256 9315 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
1257 188362 : if (op3)
1258 0 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
1259 188362 : }
1260 :
1261 : /* Helper function called by vect_finish_replace_stmt and
1262 : vect_finish_stmt_generation. Set the location of the new
1263 : statement and create and return a stmt_vec_info for it. */
1264 :
1265 : static void
1266 1447663 : vect_finish_stmt_generation_1 (vec_info *,
1267 : stmt_vec_info stmt_info, gimple *vec_stmt)
1268 : {
1269 1447663 : if (dump_enabled_p ())
1270 147858 : dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
1271 :
1272 1447663 : if (stmt_info)
1273 : {
1274 1416391 : gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
1275 :
1276 : /* While EH edges will generally prevent vectorization, stmt might
1277 : e.g. be in a must-not-throw region. Ensure newly created stmts
1278 : that could throw are part of the same region. */
1279 1416391 : int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
1280 1416391 : if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
1281 48 : add_stmt_to_eh_lp (vec_stmt, lp_nr);
1282 : }
1283 : else
1284 31272 : gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
1285 1447663 : }
1286 :
1287 : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1288 : which sets the same scalar result as STMT_INFO did. Create and return a
1289 : stmt_vec_info for VEC_STMT. */
1290 :
1291 : void
1292 916 : vect_finish_replace_stmt (vec_info *vinfo,
1293 : stmt_vec_info stmt_info, gimple *vec_stmt)
1294 : {
1295 916 : gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
1296 916 : gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
1297 :
1298 916 : gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
1299 916 : gsi_replace (&gsi, vec_stmt, true);
1300 :
1301 916 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1302 916 : }
1303 :
1304 : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1305 : before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1306 :
1307 : void
1308 1446747 : vect_finish_stmt_generation (vec_info *vinfo,
1309 : stmt_vec_info stmt_info, gimple *vec_stmt,
1310 : gimple_stmt_iterator *gsi)
1311 : {
1312 1446747 : gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
1313 :
1314 1446747 : if (!gsi_end_p (*gsi)
1315 2892267 : && gimple_has_mem_ops (vec_stmt))
1316 : {
1317 1445520 : gimple *at_stmt = gsi_stmt (*gsi);
1318 1445520 : tree vuse = gimple_vuse (at_stmt);
1319 1438908 : if (vuse && TREE_CODE (vuse) == SSA_NAME)
1320 : {
1321 1297318 : tree vdef = gimple_vdef (at_stmt);
1322 1297318 : gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1323 1297318 : gimple_set_modified (vec_stmt, true);
1324 : /* If we have an SSA vuse and insert a store, update virtual
1325 : SSA form to avoid triggering the renamer. Do so only
1326 : if we can easily see all uses - which is what almost always
1327 : happens with the way vectorized stmts are inserted. */
1328 761648 : if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1329 2058930 : && ((is_gimple_assign (vec_stmt)
1330 760766 : && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1331 65087 : || (is_gimple_call (vec_stmt)
1332 846 : && (!(gimple_call_flags (vec_stmt)
1333 846 : & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
1334 1 : || (gimple_call_lhs (vec_stmt)
1335 1 : && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
1336 : {
1337 697370 : tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1338 697370 : gimple_set_vdef (vec_stmt, new_vdef);
1339 697370 : SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1340 : }
1341 : }
1342 : }
1343 1446747 : gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1344 1446747 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1345 1446747 : }
1346 :
1347 : /* We want to vectorize a call to combined function CFN with function
1348 : decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1349 : as the types of all inputs. Check whether this is possible using
1350 : an internal function, returning its code if so or IFN_LAST if not. */
1351 :
1352 : static internal_fn
1353 16285 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
1354 : tree vectype_out, tree vectype_in)
1355 : {
1356 16285 : internal_fn ifn;
1357 16285 : if (internal_fn_p (cfn))
1358 13805 : ifn = as_internal_fn (cfn);
1359 : else
1360 2480 : ifn = associated_internal_fn (fndecl);
1361 16285 : if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1362 : {
1363 12877 : const direct_internal_fn_info &info = direct_internal_fn (ifn);
1364 12877 : if (info.vectorizable)
1365 : {
1366 12877 : bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
1367 12877 : tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1368 12877 : tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1369 :
1370 : /* The type size of both the vectype_in and vectype_out should be
1371 : exactly the same when vectype_out isn't participating the optab.
1372 : While there is no restriction for type size when vectype_out
1373 : is part of the optab query. */
1374 12877 : if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
1375 : return IFN_LAST;
1376 :
1377 12843 : if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1378 : OPTIMIZE_FOR_SPEED))
1379 : return ifn;
1380 : }
1381 : }
1382 : return IFN_LAST;
1383 : }
1384 :
1385 :
1386 : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
1387 : gimple_stmt_iterator *);
1388 :
1389 : /* Check whether a load or store statement in the loop described by
1390 : LOOP_VINFO is possible in a loop using partial vectors. This is
1391 : testing whether the vectorizer pass has the appropriate support,
1392 : as well as whether the target does.
1393 :
1394 : VLS_TYPE says whether the statement is a load or store and VECTYPE
1395 : is the type of the vector being loaded or stored. SLP_NODE is the SLP
1396 : node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1397 : says how the load or store is going to be implemented and GROUP_SIZE
1398 : is the number of load or store statements in the containing group.
1399 : If the access is a gather load or scatter store, GS_INFO describes
1400 : its arguments. If the load or store is conditional, SCALAR_MASK is the
1401 : condition under which it occurs.
1402 :
1403 : Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1404 : vectors is not supported, otherwise record the required rgroup control
1405 : types.
1406 :
1407 : If partial vectors can be used and ELSVALS is nonzero the supported
1408 : else values will be added to the vector ELSVALS points to. */
1409 :
1410 : static void
1411 293718 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
1412 : slp_tree slp_node,
1413 : vec_load_store_type vls_type,
1414 : int group_size,
1415 : vect_load_store_data *ls,
1416 : slp_tree mask_node,
1417 : vec<int> *elsvals = nullptr)
1418 : {
1419 293718 : vect_memory_access_type memory_access_type = ls->memory_access_type;
1420 :
1421 : /* Invariant loads need no special support. */
1422 293718 : if (memory_access_type == VMAT_INVARIANT)
1423 29124 : return;
1424 :
1425 : /* Figure whether the mask is uniform. scalar_mask is used to
1426 : populate the scalar_cond_masked_set. */
1427 292539 : tree scalar_mask = NULL_TREE;
1428 292539 : if (mask_node)
1429 4930 : for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
1430 : {
1431 2496 : tree def = vect_get_slp_scalar_def (mask_node, i);
1432 2496 : if (!def
1433 2496 : || (scalar_mask && def != scalar_mask))
1434 : {
1435 : scalar_mask = NULL;
1436 : break;
1437 : }
1438 : else
1439 2475 : scalar_mask = def;
1440 : }
1441 :
1442 292539 : unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
1443 292539 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1444 292539 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
1445 292539 : machine_mode vecmode = TYPE_MODE (vectype);
1446 292539 : bool is_load = (vls_type == VLS_LOAD);
1447 292539 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
1448 : {
1449 0 : nvectors /= group_size;
1450 0 : internal_fn ifn
1451 0 : = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
1452 : elsvals)
1453 0 : : vect_store_lanes_supported (vectype, group_size, true));
1454 0 : if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
1455 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1456 0 : else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
1457 0 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1458 : scalar_mask);
1459 : else
1460 : {
1461 0 : if (dump_enabled_p ())
1462 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1463 : "can't operate on partial vectors because"
1464 : " the target doesn't have an appropriate"
1465 : " load/store-lanes instruction.\n");
1466 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1467 : }
1468 0 : return;
1469 : }
1470 :
1471 292539 : if (mat_gather_scatter_p (memory_access_type))
1472 : {
1473 1735 : internal_fn ifn = (is_load
1474 1735 : ? IFN_MASK_GATHER_LOAD
1475 : : IFN_MASK_SCATTER_STORE);
1476 419 : internal_fn len_ifn = (is_load
1477 : ? IFN_MASK_LEN_GATHER_LOAD
1478 : : IFN_MASK_LEN_SCATTER_STORE);
1479 1735 : stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
1480 1735 : tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
1481 1735 : ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
1482 1735 : : ls->strided_offset_vectype);
1483 1735 : tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
1484 1735 : int scale = SLP_TREE_GS_SCALE (slp_node);
1485 :
1486 : /* The following "supported" checks just verify what we established in
1487 : get_load_store_type and don't try different offset types.
1488 : Therefore, off_vectype must be a supported offset type. In case
1489 : we chose a different one use this instead. */
1490 1735 : if (ls->supported_offset_vectype)
1491 0 : off_vectype = ls->supported_offset_vectype;
1492 : /* Same for scale. */
1493 1735 : if (ls->supported_scale)
1494 0 : scale = ls->supported_scale;
1495 :
1496 1735 : if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
1497 : memory_type,
1498 : off_vectype, scale,
1499 : elsvals))
1500 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1501 1735 : else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
1502 : memory_type,
1503 : off_vectype, scale,
1504 : elsvals)
1505 1735 : || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
1506 572 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1507 : scalar_mask);
1508 : else
1509 : {
1510 1163 : if (dump_enabled_p ())
1511 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1512 : "can't operate on partial vectors because"
1513 : " the target doesn't have an appropriate"
1514 : " gather load or scatter store instruction.\n");
1515 1163 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1516 : }
1517 1735 : return;
1518 : }
1519 :
1520 290804 : if (memory_access_type != VMAT_CONTIGUOUS)
1521 : {
1522 : /* Element X of the data must come from iteration i * VF + X of the
1523 : scalar loop. We need more work to support other mappings. */
1524 26210 : if (dump_enabled_p ())
1525 726 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1526 : "can't operate on partial vectors because an"
1527 : " access isn't contiguous.\n");
1528 26210 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1529 26210 : return;
1530 : }
1531 :
1532 264594 : if (!VECTOR_MODE_P (vecmode))
1533 : {
1534 0 : if (dump_enabled_p ())
1535 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1536 : "can't operate on partial vectors when emulating"
1537 : " vector operations.\n");
1538 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1539 0 : return;
1540 : }
1541 :
1542 : /* We might load more scalars than we need for permuting SLP loads.
1543 : We checked in get_load_store_type that the extra elements
1544 : don't leak into a new vector. */
1545 354342 : auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
1546 : {
1547 89748 : unsigned int nvectors;
1548 89748 : if (can_div_away_from_zero_p (size, nunits, &nvectors))
1549 89748 : return nvectors;
1550 : gcc_unreachable ();
1551 : };
1552 :
1553 264594 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1554 264594 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1555 264594 : machine_mode mask_mode;
1556 264594 : machine_mode vmode;
1557 264594 : bool using_partial_vectors_p = false;
1558 264594 : if (get_len_load_store_mode
1559 264594 : (vecmode, is_load, nullptr, elsvals).exists (&vmode))
1560 : {
1561 0 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1562 0 : unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
1563 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
1564 0 : using_partial_vectors_p = true;
1565 : }
1566 354342 : else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
1567 264594 : && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
1568 : elsvals))
1569 : {
1570 89748 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1571 89748 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
1572 89748 : using_partial_vectors_p = true;
1573 : }
1574 :
1575 89748 : if (!using_partial_vectors_p)
1576 : {
1577 174846 : if (dump_enabled_p ())
1578 11673 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1579 : "can't operate on partial vectors because the"
1580 : " target doesn't have the appropriate partial"
1581 : " vectorization load or store.\n");
1582 174846 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1583 : }
1584 : }
1585 :
1586 : /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1587 : form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1588 : that needs to be applied to all loads and stores in a vectorized loop.
1589 : Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1590 : otherwise return VEC_MASK & LOOP_MASK.
1591 :
1592 : MASK_TYPE is the type of both masks. If new statements are needed,
1593 : insert them before GSI. */
1594 :
1595 : tree
1596 1629 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
1597 : tree vec_mask, gimple_stmt_iterator *gsi)
1598 : {
1599 1629 : gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1600 1629 : if (!loop_mask)
1601 : return vec_mask;
1602 :
1603 139 : gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1604 :
1605 139 : if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
1606 : return vec_mask;
1607 :
1608 139 : tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1609 139 : gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1610 : vec_mask, loop_mask);
1611 :
1612 139 : gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1613 139 : return and_res;
1614 : }
1615 :
1616 : /* Determine whether we can use a gather load or scatter store to vectorize
1617 : strided load or store STMT_INFO by truncating the current offset to a
1618 : smaller width. We need to be able to construct an offset vector:
1619 :
1620 : { 0, X, X*2, X*3, ... }
1621 :
1622 : without loss of precision, where X is STMT_INFO's DR_STEP.
1623 :
1624 : Return true if this is possible, describing the gather load or scatter
1625 : store in GS_INFO. MASKED_P is true if the load or store is conditional.
1626 :
1627 : If we can use gather/scatter and ELSVALS is nonzero the supported
1628 : else values will be stored in the vector ELSVALS points to. */
1629 :
1630 : static bool
1631 63954 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
1632 : loop_vec_info loop_vinfo, bool masked_p,
1633 : gather_scatter_info *gs_info,
1634 : vec<int> *elsvals)
1635 : {
1636 63954 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1637 63954 : data_reference *dr = dr_info->dr;
1638 63954 : tree step = DR_STEP (dr);
1639 63954 : if (TREE_CODE (step) != INTEGER_CST)
1640 : {
1641 : /* ??? Perhaps we could use range information here? */
1642 28591 : if (dump_enabled_p ())
1643 201 : dump_printf_loc (MSG_NOTE, vect_location,
1644 : "cannot truncate variable step.\n");
1645 28591 : return false;
1646 : }
1647 :
1648 : /* Get the number of bits in an element. */
1649 35363 : scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1650 35363 : unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1651 :
1652 : /* Set COUNT to the upper limit on the number of elements - 1.
1653 : Start with the maximum vectorization factor. */
1654 35363 : unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1655 :
1656 : /* Try lowering COUNT to the number of scalar latch iterations. */
1657 35363 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1658 35363 : widest_int max_iters;
1659 35363 : if (max_loop_iterations (loop, &max_iters)
1660 69996 : && max_iters < count)
1661 2105 : count = max_iters.to_shwi ();
1662 :
1663 : /* Try scales of 1 and the element size. */
1664 35363 : unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
1665 35363 : wi::overflow_type overflow = wi::OVF_NONE;
1666 106089 : for (int i = 0; i < 2; ++i)
1667 : {
1668 70726 : unsigned int scale = scales[i];
1669 70726 : widest_int factor;
1670 70726 : if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1671 0 : continue;
1672 :
1673 : /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1674 70726 : widest_int range = wi::mul (count, factor, SIGNED, &overflow);
1675 70726 : if (overflow)
1676 0 : continue;
1677 70726 : signop sign = range >= 0 ? UNSIGNED : SIGNED;
1678 70726 : unsigned int min_offset_bits = wi::min_precision (range, sign);
1679 :
1680 : /* Find the narrowest viable offset type. */
1681 70726 : unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
1682 70726 : tree offset_type = build_nonstandard_integer_type (offset_bits,
1683 : sign == UNSIGNED);
1684 :
1685 : /* See whether the target supports the operation with an offset
1686 : no narrower than OFFSET_TYPE. */
1687 70726 : tree memory_type = TREE_TYPE (DR_REF (dr));
1688 70726 : tree tmp_offset_vectype;
1689 70726 : int tmp_scale;
1690 70726 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
1691 : vectype, memory_type, offset_type,
1692 : scale, &tmp_scale,
1693 : &gs_info->ifn, &gs_info->offset_vectype,
1694 : &tmp_offset_vectype, elsvals)
1695 70726 : || gs_info->ifn == IFN_LAST)
1696 70726 : continue;
1697 :
1698 0 : gs_info->decl = NULL_TREE;
1699 : /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1700 : but we don't need to store that here. */
1701 0 : gs_info->base = NULL_TREE;
1702 0 : gs_info->alias_ptr = build_int_cst
1703 0 : (reference_alias_ptr_type (DR_REF (dr)),
1704 0 : get_object_alignment (DR_REF (dr)));
1705 0 : gs_info->element_type = TREE_TYPE (vectype);
1706 0 : gs_info->offset = fold_convert (offset_type, step);
1707 0 : gs_info->scale = scale;
1708 0 : gs_info->memory_type = memory_type;
1709 0 : return true;
1710 141452 : }
1711 :
1712 35363 : if (overflow && dump_enabled_p ())
1713 0 : dump_printf_loc (MSG_NOTE, vect_location,
1714 : "truncating gather/scatter offset to %d bits"
1715 : " might change its value.\n", element_bits);
1716 :
1717 : return false;
1718 35363 : }
1719 :
1720 : /* Return true if we can use gather/scatter or strided internal functions
1721 : to vectorize STMT_INFO, which is a grouped or strided load or store
1722 : with multiple lanes and will be implemented by a type-punned access
1723 : of a vector with element size that matches the number of lanes.
1724 :
1725 : MASKED_P is true if load or store is conditional.
1726 : When returning true, fill in GS_INFO with the information required to
1727 : perform the operation. Also, store the punning type in PUNNED_VECTYPE.
1728 :
1729 : If successful and ELSVALS is nonzero the supported
1730 : else values will be stored in the vector ELSVALS points to. */
1731 :
1732 : static bool
1733 4611 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
1734 : loop_vec_info loop_vinfo, bool masked_p,
1735 : unsigned int nelts,
1736 : gather_scatter_info *info, vec<int> *elsvals,
1737 : tree *pun_vectype)
1738 : {
1739 4611 : data_reference *dr = dr_info->dr;
1740 :
1741 : /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
1742 : multiple gathers/scatter. */
1743 8895 : if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
1744 : return false;
1745 :
1746 : /* Pun the vectype with one of the same size but an element spanning
1747 : NELTS elements of VECTYPE.
1748 : The punned type of a V16QI with NELTS = 4 would be V4SI.
1749 : */
1750 3973 : tree tmp;
1751 3973 : unsigned int pieces;
1752 3973 : if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
1753 3973 : || pieces <= 1)
1754 1985 : return false;
1755 :
1756 1988 : *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
1757 :
1758 1988 : if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
1759 : return false;
1760 :
1761 1842 : internal_fn ifn;
1762 1842 : tree offset_vectype = *pun_vectype;
1763 :
1764 1261 : internal_fn strided_ifn = DR_IS_READ (dr)
1765 1842 : ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
1766 :
1767 : /* Check if we have a gather/scatter with the new type. We're just trying
1768 : with the type itself as offset for now. If not, check if we have a
1769 : strided load/store. These have fewer constraints (for example no offset
1770 : type must exist) so it is possible that even though a gather/scatter is
1771 : not available we still have a strided load/store. */
1772 1842 : bool ok = false;
1773 1842 : tree tmp_vectype;
1774 1842 : int tmp_scale;
1775 1842 : if (vect_gather_scatter_fn_p
1776 1842 : (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
1777 1842 : TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
1778 : &offset_vectype, &tmp_vectype, elsvals))
1779 : ok = true;
1780 1842 : else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
1781 : elsvals))
1782 : {
1783 : /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
1784 : will switch back to the strided variants. */
1785 0 : ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
1786 : IFN_MASK_LEN_SCATTER_STORE;
1787 0 : ok = true;
1788 : }
1789 :
1790 0 : if (ok)
1791 : {
1792 0 : info->ifn = ifn;
1793 0 : info->decl = NULL_TREE;
1794 0 : info->base = dr->ref;
1795 0 : info->alias_ptr = build_int_cst
1796 0 : (reference_alias_ptr_type (DR_REF (dr)),
1797 0 : get_object_alignment (DR_REF (dr)));
1798 0 : info->element_type = TREE_TYPE (*pun_vectype);
1799 0 : info->offset_vectype = offset_vectype;
1800 : /* No need to set the offset, vect_get_strided_load_store_ops
1801 : will do that. */
1802 0 : info->scale = 1;
1803 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
1804 0 : return true;
1805 : }
1806 :
1807 : return false;
1808 : }
1809 :
1810 :
1811 : /* Return true if we can use gather/scatter internal functions to
1812 : vectorize STMT_INFO, which is a grouped or strided load or store.
1813 : MASKED_P is true if load or store is conditional. When returning
1814 : true, fill in GS_INFO with the information required to perform the
1815 : operation.
1816 :
1817 : If we can use gather/scatter and ELSVALS is nonzero the supported
1818 : else values will be stored in the vector ELSVALS points to. */
1819 :
1820 : static bool
1821 63954 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
1822 : loop_vec_info loop_vinfo, bool masked_p,
1823 : gather_scatter_info *gs_info,
1824 : vec<int> *elsvals,
1825 : unsigned int group_size,
1826 : bool single_element_p)
1827 : {
1828 63954 : if (!vect_check_gather_scatter (stmt_info, vectype,
1829 : loop_vinfo, gs_info, elsvals)
1830 63954 : || gs_info->ifn == IFN_LAST)
1831 : {
1832 63954 : if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
1833 : masked_p, gs_info, elsvals))
1834 : return false;
1835 : }
1836 :
1837 0 : if (!single_element_p
1838 0 : && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
1839 : gs_info->scale,
1840 : group_size))
1841 : return false;
1842 :
1843 0 : if (dump_enabled_p ())
1844 0 : dump_printf_loc (MSG_NOTE, vect_location,
1845 : "using gather/scatter for strided/grouped access,"
1846 : " scale = %d\n", gs_info->scale);
1847 :
1848 : return true;
1849 : }
1850 :
1851 : /* STMT_INFO is a non-strided load or store, meaning that it accesses
1852 : elements with a known constant step. Return -1 if that step
1853 : is negative, 0 if it is zero, and 1 if it is greater than zero. */
1854 :
1855 : int
1856 1498213 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
1857 : {
1858 1498213 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1859 1498213 : return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
1860 1498213 : size_zero_node);
1861 : }
1862 :
1863 : /* If the target supports a permute mask that reverses the elements in
1864 : a vector of type VECTYPE, return that mask, otherwise return null. */
1865 :
1866 : tree
1867 9160 : perm_mask_for_reverse (tree vectype)
1868 : {
1869 9160 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1870 :
1871 : /* The encoding has a single stepped pattern. */
1872 9160 : vec_perm_builder sel (nunits, 1, 3);
1873 36640 : for (int i = 0; i < 3; ++i)
1874 27480 : sel.quick_push (nunits - 1 - i);
1875 :
1876 9160 : vec_perm_indices indices (sel, 1, nunits);
1877 9160 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
1878 : indices))
1879 : return NULL_TREE;
1880 8008 : return vect_gen_perm_mask_checked (vectype, indices);
1881 9160 : }
1882 :
1883 : /* A subroutine of get_load_store_type, with a subset of the same
1884 : arguments. Handle the case where STMT_INFO is a load or store that
1885 : accesses consecutive elements with a negative step. Sets *POFFSET
1886 : to the offset to be applied to the DR for the first access. */
1887 :
1888 : static vect_memory_access_type
1889 12282 : get_negative_load_store_type (vec_info *vinfo,
1890 : stmt_vec_info stmt_info, tree vectype,
1891 : vec_load_store_type vls_type,
1892 : unsigned int ncopies, poly_int64 *poffset)
1893 : {
1894 12282 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1895 12282 : dr_alignment_support alignment_support_scheme;
1896 :
1897 12282 : if (ncopies > 1)
1898 : {
1899 0 : if (dump_enabled_p ())
1900 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1901 : "multiple types with negative step.\n");
1902 0 : return VMAT_ELEMENTWISE;
1903 : }
1904 :
1905 : /* For backward running DRs the first access in vectype actually is
1906 : N-1 elements before the address of the DR. */
1907 12282 : *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
1908 12282 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1909 :
1910 12282 : int misalignment = dr_misalignment (dr_info, vectype, *poffset);
1911 12282 : alignment_support_scheme
1912 12282 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
1913 12282 : if (alignment_support_scheme != dr_aligned
1914 12282 : && alignment_support_scheme != dr_unaligned_supported)
1915 : {
1916 4508 : if (dump_enabled_p ())
1917 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1918 : "negative step but alignment required.\n");
1919 4508 : *poffset = 0;
1920 4508 : return VMAT_ELEMENTWISE;
1921 : }
1922 :
1923 7774 : if (vls_type == VLS_STORE_INVARIANT)
1924 : {
1925 1197 : if (dump_enabled_p ())
1926 21 : dump_printf_loc (MSG_NOTE, vect_location,
1927 : "negative step with invariant source;"
1928 : " no permute needed.\n");
1929 1197 : return VMAT_CONTIGUOUS_DOWN;
1930 : }
1931 :
1932 6577 : if (!perm_mask_for_reverse (vectype))
1933 : {
1934 1152 : if (dump_enabled_p ())
1935 52 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1936 : "negative step and reversing not supported.\n");
1937 1152 : *poffset = 0;
1938 1152 : return VMAT_ELEMENTWISE;
1939 : }
1940 :
1941 : return VMAT_CONTIGUOUS_REVERSE;
1942 : }
1943 :
1944 : /* STMT_INFO is either a masked or unconditional store. Return the value
1945 : being stored. */
1946 :
1947 : tree
1948 0 : vect_get_store_rhs (stmt_vec_info stmt_info)
1949 : {
1950 0 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
1951 : {
1952 0 : gcc_assert (gimple_assign_single_p (assign));
1953 0 : return gimple_assign_rhs1 (assign);
1954 : }
1955 0 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
1956 : {
1957 0 : internal_fn ifn = gimple_call_internal_fn (call);
1958 0 : int index = internal_fn_stored_value_index (ifn);
1959 0 : gcc_assert (index >= 0);
1960 0 : return gimple_call_arg (call, index);
1961 : }
1962 0 : gcc_unreachable ();
1963 : }
1964 :
1965 : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1966 :
1967 : This function returns a vector type which can be composed with NELTS pieces,
1968 : whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1969 : same vector size as the return vector. It checks target whether supports
1970 : pieces-size vector mode for construction firstly, if target fails to, check
1971 : pieces-size scalar mode for construction further. It returns NULL_TREE if
1972 : fails to find the available composition. If the caller only wants scalar
1973 : pieces where PTYPE e.g. is a possible gather/scatter element type
1974 : SCALAR_PTYPE_ONLY must be true.
1975 :
1976 : For example, for (vtype=V16QI, nelts=4), we can probably get:
1977 : - V16QI with PTYPE V4QI.
1978 : - V4SI with PTYPE SI.
1979 : - NULL_TREE. */
1980 :
1981 : static tree
1982 12600 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
1983 : bool scalar_ptype_only)
1984 : {
1985 12600 : gcc_assert (VECTOR_TYPE_P (vtype));
1986 12600 : gcc_assert (known_gt (nelts, 0U));
1987 :
1988 12600 : machine_mode vmode = TYPE_MODE (vtype);
1989 12600 : if (!VECTOR_MODE_P (vmode))
1990 : return NULL_TREE;
1991 :
1992 : /* When we are asked to compose the vector from its components let
1993 : that happen directly. */
1994 12600 : if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
1995 : {
1996 6017 : *ptype = TREE_TYPE (vtype);
1997 6017 : return vtype;
1998 : }
1999 :
2000 13166 : poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
2001 6583 : unsigned int pbsize;
2002 6583 : if (constant_multiple_p (vbsize, nelts, &pbsize))
2003 : {
2004 : /* First check if vec_init optab supports construction from
2005 : vector pieces directly. */
2006 6583 : scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
2007 13166 : poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
2008 6583 : machine_mode rmode;
2009 6583 : if (!scalar_ptype_only
2010 4595 : && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
2011 10716 : && (convert_optab_handler (vec_init_optab, vmode, rmode)
2012 : != CODE_FOR_nothing))
2013 : {
2014 3493 : *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
2015 3493 : return vtype;
2016 : }
2017 :
2018 : /* Otherwise check if exists an integer type of the same piece size and
2019 : if vec_init optab supports construction from it directly. */
2020 3090 : if (int_mode_for_size (pbsize, 0).exists (&elmode)
2021 3090 : && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
2022 : {
2023 2906 : if (scalar_ptype_only
2024 2906 : || convert_optab_handler (vec_init_optab, rmode, elmode)
2025 : != CODE_FOR_nothing)
2026 : {
2027 2906 : *ptype = build_nonstandard_integer_type (pbsize, 1);
2028 2906 : return build_vector_type (*ptype, nelts);
2029 : }
2030 : }
2031 : }
2032 :
2033 : return NULL_TREE;
2034 : }
2035 :
2036 : /* Check if the load permutation of NODE only refers to a consecutive
2037 : subset of the group indices where GROUP_SIZE is the size of the
2038 : dataref's group. We also assert that the length of the permutation
2039 : divides the group size and is a power of two.
2040 : Such load permutations can be elided in strided access schemes as
2041 : we can "jump over" the gap they leave. */
2042 :
2043 : bool
2044 45087 : has_consecutive_load_permutation (slp_tree node, unsigned group_size)
2045 : {
2046 45087 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
2047 45087 : if (!perm.exists ()
2048 2164 : || perm.length () <= 1
2049 496 : || !pow2p_hwi (perm.length ())
2050 45567 : || group_size % perm.length ())
2051 : return false;
2052 :
2053 433 : return vect_load_perm_consecutive_p (node);
2054 : }
2055 :
2056 :
2057 : /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
2058 : if there is a memory access type that the vectorized form can use,
2059 : storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2060 : or scatters, fill in GS_INFO accordingly. In addition
2061 : *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2062 : the target does not support the alignment scheme. *MISALIGNMENT
2063 : is set according to the alignment of the access (including
2064 : DR_MISALIGNMENT_UNKNOWN when it is unknown).
2065 :
2066 : MASKED_P is true if the statement is conditional on a vectorized mask.
2067 : VECTYPE is the vector type that the vectorized statements will use.
2068 :
2069 : If ELSVALS is nonzero the supported else values will be stored in the
2070 : vector ELSVALS points to. */
2071 :
2072 : static bool
2073 1383624 : get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
2074 : tree vectype, slp_tree slp_node,
2075 : bool masked_p, vec_load_store_type vls_type,
2076 : vect_load_store_data *ls)
2077 : {
2078 1383624 : vect_memory_access_type *memory_access_type = &ls->memory_access_type;
2079 1383624 : poly_int64 *poffset = &ls->poffset;
2080 1383624 : dr_alignment_support *alignment_support_scheme
2081 : = &ls->alignment_support_scheme;
2082 1383624 : int *misalignment = &ls->misalignment;
2083 1383624 : internal_fn *lanes_ifn = &ls->lanes_ifn;
2084 1383624 : vec<int> *elsvals = &ls->elsvals;
2085 1383624 : tree *ls_type = &ls->ls_type;
2086 1383624 : bool *slp_perm = &ls->slp_perm;
2087 1383624 : unsigned *n_perms = &ls->n_perms;
2088 1383624 : unsigned *n_loads = &ls->n_loads;
2089 1383624 : tree *supported_offset_vectype = &ls->supported_offset_vectype;
2090 1383624 : int *supported_scale = &ls->supported_scale;
2091 1383624 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
2092 1383624 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2093 1383624 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2094 1383624 : stmt_vec_info first_stmt_info;
2095 1383624 : unsigned int group_size;
2096 1383624 : unsigned HOST_WIDE_INT gap;
2097 1383624 : bool single_element_p;
2098 1383624 : poly_int64 neg_ldst_offset = 0;
2099 :
2100 1383624 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2101 1383624 : *poffset = 0;
2102 1383624 : *ls_type = NULL_TREE;
2103 1383624 : *slp_perm = false;
2104 1383624 : *n_perms = -1U;
2105 1383624 : *n_loads = -1U;
2106 1383624 : ls->subchain_p = false;
2107 :
2108 1383624 : bool perm_ok = true;
2109 1383624 : poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
2110 :
2111 1383624 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2112 73038 : perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
2113 73038 : vf, true, n_perms, n_loads);
2114 :
2115 1383624 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2116 : {
2117 885491 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
2118 885491 : group_size = DR_GROUP_SIZE (first_stmt_info);
2119 885491 : gap = DR_GROUP_GAP (first_stmt_info);
2120 885491 : single_element_p = (stmt_info == first_stmt_info
2121 885491 : && !DR_GROUP_NEXT_ELEMENT (stmt_info));
2122 : }
2123 : else
2124 : {
2125 : first_stmt_info = stmt_info;
2126 : group_size = 1;
2127 : gap = 0;
2128 : single_element_p = true;
2129 : }
2130 1383624 : dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
2131 :
2132 : /* True if the vectorized statements would access beyond the last
2133 : statement in the group. */
2134 1383624 : bool overrun_p = false;
2135 :
2136 : /* True if we can cope with such overrun by peeling for gaps, so that
2137 : there is at least one final scalar iteration after the vector loop. */
2138 2767248 : bool can_overrun_p = (!masked_p
2139 1383624 : && vls_type == VLS_LOAD
2140 554672 : && loop_vinfo
2141 1806089 : && !loop->inner);
2142 :
2143 : /* There can only be a gap at the end of the group if the stride is
2144 : known at compile time. */
2145 1383624 : gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
2146 :
2147 : /* For SLP vectorization we directly vectorize a subchain
2148 : without permutation. */
2149 1383624 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2150 1310586 : first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
2151 :
2152 1383624 : if (STMT_VINFO_STRIDED_P (first_stmt_info))
2153 : {
2154 : /* Try to use consecutive accesses of as many elements as possible,
2155 : separated by the stride, until we have a complete vector.
2156 : Fall back to scalar accesses if that isn't possible. */
2157 45087 : *memory_access_type = VMAT_STRIDED_SLP;
2158 :
2159 : /* If the load permutation is consecutive we can reduce the group to
2160 : the elements the permutation accesses. Then we release the
2161 : permutation. */
2162 45087 : if (has_consecutive_load_permutation (slp_node, group_size))
2163 : {
2164 32 : ls->subchain_p = true;
2165 32 : group_size = SLP_TREE_LANES (slp_node);
2166 32 : SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
2167 : }
2168 : }
2169 1338537 : else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2170 : {
2171 10892 : slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
2172 10892 : tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
2173 10892 : int scale = SLP_TREE_GS_SCALE (slp_node);
2174 10892 : tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
2175 10892 : tree tem;
2176 10892 : if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
2177 : masked_p, vectype, memory_type,
2178 : offset_vectype, scale, supported_scale,
2179 : &ls->gs.ifn, &tem,
2180 : supported_offset_vectype, elsvals))
2181 : {
2182 0 : if (dump_enabled_p ())
2183 : {
2184 0 : dump_printf_loc (MSG_NOTE, vect_location,
2185 : "gather/scatter with required "
2186 : "offset type "
2187 : "%T and offset scale %d.\n",
2188 : offset_vectype, scale);
2189 0 : if (*supported_offset_vectype)
2190 0 : dump_printf_loc (MSG_NOTE, vect_location,
2191 : " target supports offset type %T.\n",
2192 : *supported_offset_vectype);
2193 0 : if (*supported_scale)
2194 0 : dump_printf_loc (MSG_NOTE, vect_location,
2195 : " target supports offset scale %d.\n",
2196 : *supported_scale);
2197 : }
2198 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2199 : }
2200 10892 : else if (vls_type == VLS_LOAD
2201 10892 : ? (targetm.vectorize.builtin_gather
2202 9301 : && (ls->gs.decl
2203 9301 : = targetm.vectorize.builtin_gather (vectype,
2204 9301 : TREE_TYPE
2205 : (offset_vectype),
2206 : scale)))
2207 1591 : : (targetm.vectorize.builtin_scatter
2208 1591 : && (ls->gs.decl
2209 1591 : = targetm.vectorize.builtin_scatter (vectype,
2210 1591 : TREE_TYPE
2211 : (offset_vectype),
2212 : scale))))
2213 580 : *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
2214 : else
2215 : {
2216 : /* GATHER_SCATTER_EMULATED_P. */
2217 10312 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
2218 10312 : || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
2219 10312 : || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
2220 10312 : || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
2221 10312 : TYPE_VECTOR_SUBPARTS (vectype)))
2222 : {
2223 2740 : if (dump_enabled_p ())
2224 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2225 : "unsupported vector types for emulated "
2226 : "gather.\n");
2227 2740 : return false;
2228 : }
2229 7572 : *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
2230 : }
2231 : }
2232 : else
2233 : {
2234 1327645 : int cmp = compare_step_with_zero (vinfo, stmt_info);
2235 1327645 : if (cmp < 0)
2236 : {
2237 12460 : if (single_element_p)
2238 : /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2239 : only correct for single element "interleaving" SLP. */
2240 12282 : *memory_access_type = get_negative_load_store_type
2241 12282 : (vinfo, stmt_info, vectype, vls_type, 1,
2242 : &neg_ldst_offset);
2243 : else
2244 : /* We can fall back to VMAT_STRIDED_SLP since that does
2245 : not care whether the stride between the group instances
2246 : is positive or negative. */
2247 178 : *memory_access_type = VMAT_STRIDED_SLP;
2248 : }
2249 1315185 : else if (cmp == 0 && loop_vinfo)
2250 : {
2251 3363 : gcc_assert (vls_type == VLS_LOAD);
2252 3363 : *memory_access_type = VMAT_INVARIANT;
2253 : }
2254 : /* Try using LOAD/STORE_LANES. */
2255 1311822 : else if (slp_node->ldst_lanes
2256 1311822 : && (*lanes_ifn
2257 0 : = (vls_type == VLS_LOAD
2258 0 : ? vect_load_lanes_supported (vectype, group_size,
2259 : masked_p, elsvals)
2260 0 : : vect_store_lanes_supported (vectype, group_size,
2261 : masked_p))) != IFN_LAST)
2262 0 : *memory_access_type = VMAT_LOAD_STORE_LANES;
2263 1311822 : else if (!loop_vinfo && slp_node->avoid_stlf_fail)
2264 : {
2265 70 : *memory_access_type = VMAT_ELEMENTWISE;
2266 70 : if (dump_enabled_p ())
2267 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2268 : "using element-wise load to avoid disrupting "
2269 : "cross iteration store-to-load forwarding\n");
2270 : }
2271 : else
2272 1311752 : *memory_access_type = VMAT_CONTIGUOUS;
2273 :
2274 : /* If this is single-element interleaving with an element
2275 : distance that leaves unused vector loads around fall back
2276 : to elementwise access if possible - we otherwise least
2277 : create very sub-optimal code in that case (and
2278 : blow up memory, see PR65518). */
2279 1327645 : if (loop_vinfo
2280 1327645 : && single_element_p
2281 478665 : && (*memory_access_type == VMAT_CONTIGUOUS
2282 15645 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2283 1806310 : && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
2284 : {
2285 17805 : *memory_access_type = VMAT_ELEMENTWISE;
2286 17805 : if (dump_enabled_p ())
2287 198 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2288 : "single-element interleaving not supported "
2289 : "for not adjacent vector loads, using "
2290 : "elementwise access\n");
2291 : }
2292 :
2293 : /* Also fall back to elementwise access in case we did not lower a
2294 : permutation and cannot code generate it. */
2295 1327645 : if (loop_vinfo
2296 533111 : && *memory_access_type != VMAT_ELEMENTWISE
2297 509646 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2298 1356186 : && !perm_ok)
2299 : {
2300 2062 : *memory_access_type = VMAT_ELEMENTWISE;
2301 2062 : if (dump_enabled_p ())
2302 246 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2303 : "permutation not supported, using elementwise "
2304 : "access\n");
2305 : }
2306 :
2307 533111 : overrun_p = (loop_vinfo && gap != 0
2308 1370526 : && *memory_access_type != VMAT_ELEMENTWISE);
2309 1327645 : if (overrun_p && vls_type != VLS_LOAD)
2310 : {
2311 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2312 : "Grouped store with gaps requires"
2313 : " non-consecutive accesses\n");
2314 9 : return false;
2315 : }
2316 :
2317 1327645 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
2318 1327645 : poly_int64 off = 0;
2319 1327645 : if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2320 5266 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2321 :
2322 : /* An overrun is fine if the trailing elements are smaller
2323 : than the alignment boundary B. Every vector access will
2324 : be a multiple of B and so we are guaranteed to access a
2325 : non-gap element in the same B-sized block. */
2326 1327645 : if (overrun_p
2327 1327645 : && gap < (vect_known_alignment_in_bytes (first_dr_info,
2328 22927 : vectype, off) / dr_size))
2329 : overrun_p = false;
2330 :
2331 : /* When we have a contiguous access across loop iterations
2332 : but the access in the loop doesn't cover the full vector
2333 : we can end up with no gap recorded but still excess
2334 : elements accessed, see PR103116. Make sure we peel for
2335 : gaps if necessary and sufficient and give up if not.
2336 :
2337 : If there is a combination of the access not covering the full
2338 : vector and a gap recorded then we may need to peel twice. */
2339 1327645 : bool large_vector_overrun_p = false;
2340 1327645 : if (loop_vinfo
2341 533111 : && (*memory_access_type == VMAT_CONTIGUOUS
2342 35526 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2343 502851 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2344 1353745 : && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
2345 : nunits))
2346 : large_vector_overrun_p = overrun_p = true;
2347 :
2348 : /* If the gap splits the vector in half and the target
2349 : can do half-vector operations avoid the epilogue peeling
2350 : by simply loading half of the vector only. Usually
2351 : the construction with an upper zero half will be elided. */
2352 1327645 : dr_alignment_support alss;
2353 1327645 : int misalign = dr_misalignment (first_dr_info, vectype, off);
2354 1327645 : tree half_vtype;
2355 1327645 : poly_uint64 remain;
2356 1327645 : unsigned HOST_WIDE_INT tem, num;
2357 1327645 : if (overrun_p
2358 1327645 : && !masked_p
2359 17476 : && *memory_access_type != VMAT_LOAD_STORE_LANES
2360 17476 : && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
2361 : vectype, misalign)))
2362 : == dr_aligned
2363 14976 : || alss == dr_unaligned_supported)
2364 9886 : && can_div_trunc_p (group_size
2365 9886 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2366 : nunits, &tem, &remain)
2367 1337531 : && (known_eq (remain, 0u)
2368 7407 : || (known_ne (remain, 0u)
2369 5740 : && constant_multiple_p (nunits, remain, &num)
2370 1325166 : && (vector_vector_composition_type (vectype, num, &half_vtype)
2371 : != NULL_TREE))))
2372 8219 : overrun_p = false;
2373 :
2374 1327645 : if (overrun_p && !can_overrun_p)
2375 : {
2376 6 : if (dump_enabled_p ())
2377 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2378 : "Peeling for outer loop is not supported\n");
2379 6 : return false;
2380 : }
2381 :
2382 : /* Peeling for gaps assumes that a single scalar iteration
2383 : is enough to make sure the last vector iteration doesn't
2384 : access excess elements. */
2385 1327639 : if (overrun_p
2386 1327639 : && (!can_div_trunc_p (group_size
2387 9251 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2388 : nunits, &tem, &remain)
2389 9251 : || maybe_lt (remain + group_size, nunits)))
2390 : {
2391 : /* But peeling a single scalar iteration is enough if
2392 : we can use the next power-of-two sized partial
2393 : access and that is sufficiently small to be covered
2394 : by the single scalar iteration. */
2395 16 : unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
2396 16 : if (masked_p
2397 16 : || !nunits.is_constant (&cnunits)
2398 16 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
2399 16 : || (((cremain = (group_size * cvf - gap) % cnunits), true)
2400 16 : && ((cpart_size = (1 << ceil_log2 (cremain))), true)
2401 16 : && (cremain + group_size < cpart_size
2402 13 : || (vector_vector_composition_type (vectype,
2403 13 : cnunits / cpart_size,
2404 : &half_vtype)
2405 : == NULL_TREE))))
2406 : {
2407 : /* If all fails we can still resort to niter masking unless
2408 : the vectors used are too big, so enforce the use of
2409 : partial vectors. */
2410 3 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2411 3 : && !large_vector_overrun_p)
2412 : {
2413 0 : if (dump_enabled_p ())
2414 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2415 : "peeling for gaps insufficient for "
2416 : "access unless using partial "
2417 : "vectors\n");
2418 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2419 : }
2420 : else
2421 : {
2422 3 : if (dump_enabled_p ())
2423 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2424 : "peeling for gaps insufficient for "
2425 : "access\n");
2426 3 : return false;
2427 : }
2428 : }
2429 13 : else if (large_vector_overrun_p)
2430 : {
2431 13 : if (dump_enabled_p ())
2432 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2433 : "can't operate on partial vectors because "
2434 : "only unmasked loads handle access "
2435 : "shortening required because of gaps at "
2436 : "the end of the access\n");
2437 13 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2438 : }
2439 : }
2440 : }
2441 :
2442 : /* As a last resort, trying using a gather load or scatter store.
2443 :
2444 : ??? Although the code can handle all group sizes correctly,
2445 : it probably isn't a win to use separate strided accesses based
2446 : on nearby locations. Or, even if it's a win over scalar code,
2447 : it might not be a win over vectorizing at a lower VF, if that
2448 : allows us to use contiguous accesses. */
2449 1380875 : vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
2450 1380875 : if (loop_vinfo
2451 586341 : && (*memory_access_type == VMAT_ELEMENTWISE
2452 586341 : || *memory_access_type == VMAT_STRIDED_SLP))
2453 : {
2454 70787 : gather_scatter_info gs_info;
2455 70787 : tree tem;
2456 70787 : if (SLP_TREE_LANES (slp_node) == 1
2457 65982 : && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2458 21587 : || single_element_p)
2459 134741 : && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
2460 : masked_p, &gs_info, elsvals,
2461 : group_size, single_element_p))
2462 : {
2463 : /* vect_use_strided_gather_scatters_p does not save the actually
2464 : supported scale and offset type so do that here.
2465 : We need it later in check_load_store_for_partial_vectors
2466 : where we only check if the given internal function is supported
2467 : (to choose whether to use the IFN, LEGACY, or EMULATED flavor
2468 : of gather/scatter) and don't re-do the full analysis. */
2469 0 : tree tmp;
2470 0 : gcc_assert (vect_gather_scatter_fn_p
2471 : (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
2472 : gs_info.memory_type, TREE_TYPE (gs_info.offset),
2473 : gs_info.scale, supported_scale, &gs_info.ifn,
2474 : &tmp, supported_offset_vectype, elsvals));
2475 :
2476 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2477 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2478 0 : ls->gs.ifn = gs_info.ifn;
2479 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2480 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2481 : }
2482 70787 : else if (SLP_TREE_LANES (slp_node) > 1
2483 : && !masked_p
2484 4805 : && !single_element_p
2485 75398 : && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
2486 : vectype, loop_vinfo,
2487 : masked_p, group_size,
2488 : &gs_info, elsvals, &tem))
2489 : {
2490 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2491 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2492 0 : grouped_gather_fallback = *memory_access_type;
2493 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2494 0 : ls->gs.ifn = gs_info.ifn;
2495 0 : vectype = *ls_type = tem;
2496 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2497 : }
2498 : }
2499 :
2500 1380875 : if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
2501 1380875 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2502 6459 : *poffset = neg_ldst_offset;
2503 :
2504 1380875 : if (*memory_access_type == VMAT_ELEMENTWISE
2505 1355278 : || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
2506 1354698 : || *memory_access_type == VMAT_STRIDED_SLP
2507 1309438 : || *memory_access_type == VMAT_INVARIANT)
2508 : {
2509 74800 : *alignment_support_scheme = dr_unaligned_supported;
2510 74800 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2511 : }
2512 : else
2513 : {
2514 1306075 : if (mat_gather_scatter_p (*memory_access_type)
2515 : && !first_dr_info)
2516 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2517 : else
2518 1306075 : *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
2519 1306075 : *alignment_support_scheme
2520 1306075 : = vect_supportable_dr_alignment
2521 1306075 : (vinfo, first_dr_info, vectype, *misalignment,
2522 1306075 : mat_gather_scatter_p (*memory_access_type));
2523 1306075 : if (grouped_gather_fallback != VMAT_UNINITIALIZED
2524 0 : && *alignment_support_scheme != dr_aligned
2525 0 : && *alignment_support_scheme != dr_unaligned_supported)
2526 : {
2527 : /* No supportable alignment for a grouped gather, fall back to the
2528 : original memory access type. Even though VMAT_STRIDED_SLP might
2529 : also try aligned vector loads it can still choose vector
2530 : construction from scalars. */
2531 0 : *memory_access_type = grouped_gather_fallback;
2532 0 : *alignment_support_scheme = dr_unaligned_supported;
2533 0 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2534 : }
2535 : }
2536 :
2537 1380875 : if (overrun_p)
2538 : {
2539 9248 : gcc_assert (can_overrun_p);
2540 9248 : if (dump_enabled_p ())
2541 511 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2542 : "Data access with gaps requires scalar "
2543 : "epilogue loop\n");
2544 9248 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2545 : }
2546 :
2547 1380875 : if ((*memory_access_type == VMAT_ELEMENTWISE
2548 1380875 : || *memory_access_type == VMAT_STRIDED_SLP)
2549 : && !nunits.is_constant ())
2550 : {
2551 : if (dump_enabled_p ())
2552 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2553 : "Not using elementwise accesses due to variable "
2554 : "vectorization factor.\n");
2555 : return false;
2556 : }
2557 :
2558 : /* Checks if all scalar iterations are known to be inbounds. */
2559 1380875 : bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
2560 :
2561 : /* Check if we support the operation if early breaks are needed. Here we
2562 : must ensure that we don't access any more than the scalar code would
2563 : have. A masked operation would ensure this, so for these load types
2564 : force masking. */
2565 1380875 : if (loop_vinfo
2566 586341 : && dr_safe_speculative_read_required (stmt_info)
2567 1570503 : && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2568 : {
2569 189628 : if (mat_gather_scatter_p (*memory_access_type)
2570 189628 : || *memory_access_type == VMAT_STRIDED_SLP)
2571 : {
2572 9380 : if (dump_enabled_p ())
2573 8 : dump_printf_loc (MSG_NOTE, vect_location,
2574 : "early break not supported: cannot peel for "
2575 : "alignment. With non-contiguous memory vectorization"
2576 : " could read out of bounds at %G ",
2577 : STMT_VINFO_STMT (stmt_info));
2578 9380 : if (inbounds)
2579 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2580 : else
2581 : return false;
2582 : }
2583 : /* Block-level alignment: Even though individual accesses of
2584 : VMAT_ELEMENTWISE type do not cause alignment problems, loading the
2585 : whole vector's worth of values in a speculative early-break context
2586 : might cross a page boundary. Set the alignment scheme to `dr_aligned'
2587 : here in order to force checking of whether such accesses meet
2588 : alignment criteria. */
2589 180248 : else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
2590 15003 : *alignment_support_scheme = dr_aligned;
2591 : }
2592 :
2593 : /* If this DR needs alignment for correctness, we must ensure the target
2594 : alignment is a constant power-of-two multiple of the amount read per
2595 : vector iteration or force masking. */
2596 1371495 : if (dr_safe_speculative_read_required (stmt_info)
2597 1371495 : && (*alignment_support_scheme == dr_aligned
2598 109133 : && !mat_gather_scatter_p (*memory_access_type)))
2599 : {
2600 : /* We can only peel for loops, of course. */
2601 109133 : gcc_checking_assert (loop_vinfo);
2602 :
2603 109133 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2604 109133 : poly_uint64 read_amount
2605 109133 : = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
2606 109133 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2607 109133 : read_amount *= group_size;
2608 :
2609 109133 : auto target_alignment
2610 109133 : = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
2611 109133 : if (!multiple_p (target_alignment, read_amount))
2612 : {
2613 12688 : if (dump_enabled_p ())
2614 : {
2615 28 : dump_printf_loc (MSG_NOTE, vect_location,
2616 : "desired alignment not met, target was ");
2617 28 : dump_dec (MSG_NOTE, target_alignment);
2618 28 : dump_printf (MSG_NOTE, " previously, but read amount is ");
2619 28 : dump_dec (MSG_NOTE, read_amount);
2620 28 : dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
2621 : }
2622 14922 : return false;
2623 : }
2624 :
2625 : /* When using a group access the first element may be aligned but the
2626 : subsequent loads may not be. For LOAD_LANES since the loads are based
2627 : on the first DR then all loads in the group are aligned. For
2628 : non-LOAD_LANES this is not the case. In particular a load + blend when
2629 : there are gaps can have the non first loads issued unaligned, even
2630 : partially overlapping the memory of the first load in order to simplify
2631 : the blend. This is what the x86_64 backend does for instance. As
2632 : such only the first load in the group is aligned, the rest are not.
2633 : Because of this the permutes may break the alignment requirements that
2634 : have been set, and as such we should for now, reject them. */
2635 96445 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2636 : {
2637 2234 : if (dump_enabled_p ())
2638 75 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2639 : "loads with load permutations not supported for "
2640 : "speculative early break loads for %G",
2641 : STMT_VINFO_STMT (stmt_info));
2642 2234 : return false;
2643 : }
2644 :
2645 : /* Reject vectorization if we know the read mount per vector iteration
2646 : exceeds the min page size. */
2647 94211 : if (known_gt (read_amount, (unsigned) param_min_pagesize))
2648 : {
2649 0 : if (dump_enabled_p ())
2650 : {
2651 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2652 : "alignment required for correctness (");
2653 0 : dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
2654 0 : dump_printf (MSG_NOTE, ") may exceed page size.\n");
2655 : }
2656 0 : return false;
2657 : }
2658 :
2659 94211 : if (!vf.is_constant ())
2660 : {
2661 : /* For VLA modes, we need a runtime check to ensure any speculative
2662 : read amount does not exceed the page size. Here we record the max
2663 : possible read amount for the check. */
2664 : if (maybe_gt (read_amount,
2665 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
2666 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
2667 :
2668 : /* For VLA modes, we must use partial vectors. */
2669 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2670 : }
2671 : }
2672 :
2673 1356573 : if (*alignment_support_scheme == dr_unaligned_unsupported)
2674 : {
2675 67751 : if (dump_enabled_p ())
2676 256 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2677 : "unsupported unaligned access\n");
2678 67751 : return false;
2679 : }
2680 :
2681 : /* FIXME: At the moment the cost model seems to underestimate the
2682 : cost of using elementwise accesses. This check preserves the
2683 : traditional behavior until that can be fixed. */
2684 1288822 : if (*memory_access_type == VMAT_ELEMENTWISE
2685 14867 : && !STMT_VINFO_STRIDED_P (first_stmt_info)
2686 1303689 : && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
2687 9646 : && single_element_p
2688 9003 : && !pow2p_hwi (group_size)))
2689 : {
2690 9204 : if (dump_enabled_p ())
2691 362 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2692 : "not falling back to elementwise accesses\n");
2693 9204 : return false;
2694 : }
2695 :
2696 : /* For BB vectorization build up the vector from existing scalar defs. */
2697 1279618 : if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
2698 : return false;
2699 :
2700 : /* Some loads need to explicitly permute the loaded data if there
2701 : is a load permutation. Among those are:
2702 : - VMAT_ELEMENTWISE.
2703 : - VMAT_STRIDED_SLP.
2704 : - VMAT_GATHER_SCATTER:
2705 : - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
2706 : - Grouped strided gather (ditto but for #lanes > 1).
2707 :
2708 : For VMAT_ELEMENTWISE we can fold the load permutation into the
2709 : individual indices we access directly, eliding the permutation.
2710 : Strided gather only allows load permutations for the
2711 : single-element case. */
2712 :
2713 1279618 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2714 1279618 : && !(*memory_access_type == VMAT_ELEMENTWISE
2715 46083 : || (mat_gather_scatter_p (*memory_access_type)
2716 0 : && SLP_TREE_LANES (slp_node) == 1
2717 0 : && single_element_p)))
2718 : {
2719 46083 : if (!loop_vinfo)
2720 : {
2721 : /* In BB vectorization we may not actually use a loaded vector
2722 : accessing elements in excess of DR_GROUP_SIZE. */
2723 24205 : stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
2724 24205 : group_info = DR_GROUP_FIRST_ELEMENT (group_info);
2725 24205 : unsigned HOST_WIDE_INT nunits;
2726 24205 : unsigned j, k, maxk = 0;
2727 86519 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
2728 62314 : if (k > maxk)
2729 : maxk = k;
2730 24205 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2731 43538 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
2732 24205 : || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
2733 : {
2734 4872 : if (dump_enabled_p ())
2735 29 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2736 : "BB vectorization with gaps at the end of "
2737 : "a load is not supported\n");
2738 4872 : return false;
2739 : }
2740 : }
2741 :
2742 41211 : if (!perm_ok)
2743 : {
2744 2123 : if (dump_enabled_p ())
2745 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
2746 : vect_location,
2747 : "unsupported load permutation\n");
2748 2123 : return false;
2749 : }
2750 :
2751 39088 : *slp_perm = true;
2752 : }
2753 :
2754 : return true;
2755 : }
2756 :
2757 : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2758 : conditional operation STMT_INFO. When returning true, store the mask
2759 : in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
2760 : the vectorized mask in *MASK_VECTYPE_OUT. */
2761 :
2762 : static bool
2763 12666 : vect_check_scalar_mask (vec_info *vinfo,
2764 : slp_tree slp_node, unsigned mask_index,
2765 : slp_tree *mask_node,
2766 : vect_def_type *mask_dt_out, tree *mask_vectype_out)
2767 : {
2768 12666 : enum vect_def_type mask_dt;
2769 12666 : tree mask_vectype;
2770 12666 : slp_tree mask_node_1;
2771 12666 : tree mask_;
2772 12666 : if (!vect_is_simple_use (vinfo, slp_node, mask_index,
2773 : &mask_, &mask_node_1, &mask_dt, &mask_vectype))
2774 : {
2775 0 : if (dump_enabled_p ())
2776 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2777 : "mask use not simple.\n");
2778 0 : return false;
2779 : }
2780 :
2781 12666 : if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
2782 12666 : && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
2783 : {
2784 0 : if (dump_enabled_p ())
2785 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2786 : "mask argument is not a boolean.\n");
2787 0 : return false;
2788 : }
2789 :
2790 12666 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2791 12666 : if (!mask_vectype)
2792 19 : mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
2793 : mask_node_1);
2794 :
2795 12666 : if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2796 : {
2797 0 : if (dump_enabled_p ())
2798 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2799 : "could not find an appropriate vector mask type.\n");
2800 0 : return false;
2801 : }
2802 :
2803 12666 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2804 25332 : TYPE_VECTOR_SUBPARTS (vectype)))
2805 : {
2806 0 : if (dump_enabled_p ())
2807 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2808 : "vector mask type %T"
2809 : " does not match vector data type %T.\n",
2810 : mask_vectype, vectype);
2811 :
2812 0 : return false;
2813 : }
2814 :
2815 12666 : *mask_dt_out = mask_dt;
2816 12666 : *mask_vectype_out = mask_vectype;
2817 12666 : *mask_node = mask_node_1;
2818 12666 : return true;
2819 : }
2820 :
2821 :
2822 : /* Return true if stored value is suitable for vectorizing store
2823 : statement STMT_INFO. When returning true, store the scalar stored
2824 : in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2825 : the type of the vectorized store value in
2826 : *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2827 :
2828 : static bool
2829 1379334 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
2830 : slp_tree slp_node, slp_tree *rhs_node,
2831 : vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
2832 : vec_load_store_type *vls_type_out)
2833 : {
2834 1379334 : int op_no = 0;
2835 1379334 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
2836 : {
2837 1881 : if (gimple_call_internal_p (call)
2838 1881 : && internal_store_fn_p (gimple_call_internal_fn (call)))
2839 1881 : op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
2840 : }
2841 1379334 : op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
2842 :
2843 1379334 : enum vect_def_type rhs_dt;
2844 1379334 : tree rhs_vectype;
2845 1379334 : tree rhs;
2846 1379334 : if (!vect_is_simple_use (vinfo, slp_node, op_no,
2847 : &rhs, rhs_node, &rhs_dt, &rhs_vectype))
2848 : {
2849 0 : if (dump_enabled_p ())
2850 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2851 : "use not simple.\n");
2852 0 : return false;
2853 : }
2854 :
2855 : /* In the case this is a store from a constant make sure
2856 : native_encode_expr can handle it. */
2857 1379334 : if (rhs_dt == vect_constant_def
2858 1379334 : && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2859 : {
2860 0 : if (dump_enabled_p ())
2861 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2862 : "cannot encode constant as a byte sequence.\n");
2863 0 : return false;
2864 : }
2865 :
2866 1379334 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2867 1379334 : if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2868 : {
2869 24 : if (dump_enabled_p ())
2870 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2871 : "incompatible vector types.\n");
2872 24 : return false;
2873 : }
2874 :
2875 1379310 : *rhs_dt_out = rhs_dt;
2876 1379310 : *rhs_vectype_out = rhs_vectype;
2877 1379310 : if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2878 1017478 : *vls_type_out = VLS_STORE_INVARIANT;
2879 : else
2880 361832 : *vls_type_out = VLS_STORE;
2881 : return true;
2882 : }
2883 :
2884 : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2885 : Note that we support masks with floating-point type, in which case the
2886 : floats are interpreted as a bitmask. */
2887 :
2888 : static tree
2889 170 : vect_build_all_ones_mask (vec_info *vinfo,
2890 : stmt_vec_info stmt_info, tree masktype)
2891 : {
2892 170 : if (TREE_CODE (masktype) == INTEGER_TYPE)
2893 98 : return build_int_cst (masktype, -1);
2894 72 : else if (VECTOR_BOOLEAN_TYPE_P (masktype)
2895 144 : || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2896 : {
2897 19 : tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2898 19 : mask = build_vector_from_val (masktype, mask);
2899 19 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2900 : }
2901 53 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2902 : {
2903 : REAL_VALUE_TYPE r;
2904 : long tmp[6];
2905 371 : for (int j = 0; j < 6; ++j)
2906 318 : tmp[j] = -1;
2907 53 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2908 53 : tree mask = build_real (TREE_TYPE (masktype), r);
2909 53 : mask = build_vector_from_val (masktype, mask);
2910 53 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2911 : }
2912 0 : gcc_unreachable ();
2913 : }
2914 :
2915 : /* Build an all-zero merge value of type VECTYPE while vectorizing
2916 : STMT_INFO as a gather load. */
2917 :
2918 : static tree
2919 158 : vect_build_zero_merge_argument (vec_info *vinfo,
2920 : stmt_vec_info stmt_info, tree vectype)
2921 : {
2922 158 : tree merge;
2923 158 : if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2924 49 : merge = build_int_cst (TREE_TYPE (vectype), 0);
2925 109 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2926 : {
2927 : REAL_VALUE_TYPE r;
2928 : long tmp[6];
2929 763 : for (int j = 0; j < 6; ++j)
2930 654 : tmp[j] = 0;
2931 109 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2932 109 : merge = build_real (TREE_TYPE (vectype), r);
2933 : }
2934 : else
2935 0 : gcc_unreachable ();
2936 158 : merge = build_vector_from_val (vectype, merge);
2937 158 : return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
2938 : }
2939 :
2940 : /* Return the corresponding else value for an else value constant
2941 : ELSVAL with type TYPE. */
2942 :
2943 : tree
2944 1967 : vect_get_mask_load_else (int elsval, tree type)
2945 : {
2946 1967 : tree els;
2947 1967 : if (elsval == MASK_LOAD_ELSE_UNDEFINED)
2948 : {
2949 0 : tree tmp = create_tmp_var (type);
2950 : /* No need to warn about anything. */
2951 0 : TREE_NO_WARNING (tmp) = 1;
2952 0 : els = get_or_create_ssa_default_def (cfun, tmp);
2953 : }
2954 1967 : else if (elsval == MASK_LOAD_ELSE_M1)
2955 0 : els = build_minus_one_cst (type);
2956 1967 : else if (elsval == MASK_LOAD_ELSE_ZERO)
2957 1967 : els = build_zero_cst (type);
2958 : else
2959 0 : gcc_unreachable ();
2960 :
2961 1967 : return els;
2962 : }
2963 :
2964 : /* Build a gather load call while vectorizing STMT_INFO. Insert new
2965 : instructions before GSI and add them to VEC_STMT. GS_INFO describes
2966 : the gather load operation. If the load is conditional, MASK is the
2967 : vectorized condition, otherwise MASK is null. PTR is the base
2968 : pointer and OFFSET is the vectorized offset. */
2969 :
2970 : static gimple *
2971 349 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
2972 : slp_tree slp_node, tree vectype,
2973 : gimple_stmt_iterator *gsi, tree decl,
2974 : tree ptr, tree offset, tree mask)
2975 : {
2976 349 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
2977 349 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
2978 349 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2979 349 : /* ptrtype */ arglist = TREE_CHAIN (arglist);
2980 349 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2981 349 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2982 349 : tree scaletype = TREE_VALUE (arglist);
2983 349 : tree var;
2984 349 : gcc_checking_assert (types_compatible_p (srctype, rettype)
2985 : && (!mask
2986 : || TREE_CODE (masktype) == INTEGER_TYPE
2987 : || types_compatible_p (srctype, masktype)));
2988 :
2989 349 : tree op = offset;
2990 349 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
2991 : {
2992 103 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
2993 : TYPE_VECTOR_SUBPARTS (idxtype)));
2994 103 : var = vect_get_new_ssa_name (idxtype, vect_simple_var);
2995 103 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
2996 103 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
2997 103 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
2998 103 : op = var;
2999 : }
3000 :
3001 349 : tree src_op = NULL_TREE;
3002 349 : tree mask_op = NULL_TREE;
3003 349 : if (mask)
3004 : {
3005 191 : if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
3006 : {
3007 191 : tree utype, optype = TREE_TYPE (mask);
3008 191 : if (VECTOR_TYPE_P (masktype)
3009 191 : || TYPE_MODE (masktype) == TYPE_MODE (optype))
3010 : utype = masktype;
3011 : else
3012 6 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3013 191 : var = vect_get_new_ssa_name (utype, vect_scalar_var);
3014 191 : tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
3015 191 : gassign *new_stmt
3016 191 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3017 191 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3018 191 : mask_arg = var;
3019 191 : if (!useless_type_conversion_p (masktype, utype))
3020 : {
3021 6 : gcc_assert (TYPE_PRECISION (utype)
3022 : <= TYPE_PRECISION (masktype));
3023 6 : var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3024 6 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3025 6 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3026 6 : mask_arg = var;
3027 : }
3028 191 : src_op = build_zero_cst (srctype);
3029 191 : mask_op = mask_arg;
3030 : }
3031 : else
3032 : {
3033 : src_op = mask;
3034 : mask_op = mask;
3035 : }
3036 : }
3037 : else
3038 : {
3039 158 : src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
3040 158 : mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
3041 : }
3042 :
3043 349 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3044 349 : gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
3045 : mask_op, scale);
3046 :
3047 349 : if (!useless_type_conversion_p (vectype, rettype))
3048 : {
3049 52 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
3050 : TYPE_VECTOR_SUBPARTS (rettype)));
3051 52 : op = vect_get_new_ssa_name (rettype, vect_simple_var);
3052 52 : gimple_call_set_lhs (new_stmt, op);
3053 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3054 52 : op = build1 (VIEW_CONVERT_EXPR, vectype, op);
3055 52 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
3056 : }
3057 :
3058 349 : return new_stmt;
3059 : }
3060 :
3061 : /* Build a scatter store call while vectorizing STMT_INFO. Insert new
3062 : instructions before GSI. GS_INFO describes the scatter store operation.
3063 : PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
3064 : vectorized data to store.
3065 : If the store is conditional, MASK is the vectorized condition, otherwise
3066 : MASK is null. */
3067 :
3068 : static gimple *
3069 161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
3070 : slp_tree slp_node,
3071 : gimple_stmt_iterator *gsi,
3072 : tree decl,
3073 : tree ptr, tree offset, tree oprnd, tree mask)
3074 : {
3075 161 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
3076 161 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
3077 161 : /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
3078 161 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3079 161 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3080 161 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3081 161 : tree scaletype = TREE_VALUE (arglist);
3082 161 : gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
3083 : && TREE_CODE (rettype) == VOID_TYPE);
3084 :
3085 161 : tree mask_arg = NULL_TREE;
3086 161 : if (mask)
3087 : {
3088 110 : mask_arg = mask;
3089 110 : tree optype = TREE_TYPE (mask_arg);
3090 110 : tree utype;
3091 110 : if (TYPE_MODE (masktype) == TYPE_MODE (optype))
3092 : utype = masktype;
3093 : else
3094 8 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3095 110 : tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
3096 110 : mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
3097 110 : gassign *new_stmt
3098 110 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3099 110 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3100 110 : mask_arg = var;
3101 110 : if (!useless_type_conversion_p (masktype, utype))
3102 : {
3103 8 : gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
3104 8 : tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3105 8 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3106 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3107 8 : mask_arg = var;
3108 : }
3109 : }
3110 : else
3111 : {
3112 51 : mask_arg = build_int_cst (masktype, -1);
3113 51 : mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
3114 : }
3115 :
3116 161 : tree src = oprnd;
3117 161 : if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
3118 : {
3119 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
3120 : TYPE_VECTOR_SUBPARTS (srctype)));
3121 0 : tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
3122 0 : src = build1 (VIEW_CONVERT_EXPR, srctype, src);
3123 0 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
3124 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3125 0 : src = var;
3126 : }
3127 :
3128 161 : tree op = offset;
3129 161 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3130 : {
3131 16 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3132 : TYPE_VECTOR_SUBPARTS (idxtype)));
3133 16 : tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3134 16 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3135 16 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3136 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3137 16 : op = var;
3138 : }
3139 :
3140 161 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3141 161 : gcall *new_stmt
3142 161 : = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
3143 161 : return new_stmt;
3144 : }
3145 :
3146 : /* Prepare the base and offset in GS_INFO for vectorization.
3147 : Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
3148 : to the vectorized offset argument for the first copy of STMT_INFO.
3149 : STMT_INFO is the statement described by GS_INFO and LOOP is the
3150 : containing loop. */
3151 :
3152 : static void
3153 1235 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
3154 : tree *dataref_ptr, vec<tree> *vec_offset)
3155 : {
3156 1235 : gimple_seq stmts = NULL;
3157 1235 : *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
3158 : &stmts, true, NULL_TREE);
3159 1235 : if (stmts != NULL)
3160 : {
3161 1002 : basic_block new_bb;
3162 1002 : edge pe = loop_preheader_edge (loop);
3163 1002 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3164 1002 : gcc_assert (!new_bb);
3165 : }
3166 1235 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
3167 1235 : }
3168 :
3169 : /* Prepare to implement a grouped or strided load or store using
3170 : the gather load or scatter store operation described by GS_INFO.
3171 : STMT_INFO is the load or store statement.
3172 :
3173 : Set *DR_STEP to the amount that should be added to pointer base address
3174 : to get to the next iteration's base address.
3175 : Set *DR_BUMP to the amount that should be added to the base
3176 : address after each copy of the vectorized statement in a grouped read.
3177 : Set *VEC_OFFSET to an invariant offset vector in which element I has the
3178 : value I * DR_STEP / SCALE. */
3179 :
3180 : static void
3181 0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
3182 : tree vectype, tree offset_vectype,
3183 : loop_vec_info loop_vinfo,
3184 : gimple_stmt_iterator *gsi,
3185 : tree *dr_step, tree *dr_bump,
3186 : tree *vec_offset)
3187 : {
3188 0 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
3189 :
3190 0 : tree dr_step_temp
3191 0 : = size_binop (MULT_EXPR,
3192 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3193 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
3194 0 : *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
3195 0 : ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
3196 0 : : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
3197 : true, GSI_SAME_STMT);
3198 0 : tree bump = size_binop (MULT_EXPR,
3199 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3200 : size_int (TYPE_VECTOR_SUBPARTS (vectype)));
3201 0 : *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
3202 :
3203 0 : internal_fn ifn
3204 0 : = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
3205 0 : if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
3206 : {
3207 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
3208 : unshare_expr (DR_STEP (dr)));
3209 0 : return;
3210 : }
3211 :
3212 : /* The offset given in GS_INFO can have pointer type, so use the element
3213 : type of the vector instead. */
3214 0 : tree offset_type = TREE_TYPE (offset_vectype);
3215 :
3216 : /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3217 0 : tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
3218 : ssize_int (SLP_TREE_GS_SCALE (node)));
3219 0 : step = fold_convert (offset_type, step);
3220 :
3221 : /* Create {0, X, X*2, X*3, ...}. */
3222 0 : tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
3223 : build_zero_cst (offset_type), step);
3224 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
3225 : }
3226 :
3227 : /* Return the amount that should be added to a vector pointer, represented by
3228 : DR_INFO, to increment to the next vectorized iteration. */
3229 :
3230 : static tree
3231 711464 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
3232 : vect_memory_access_type memory_access_type)
3233 : {
3234 711464 : if (memory_access_type == VMAT_INVARIANT)
3235 0 : return size_zero_node;
3236 :
3237 711464 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3238 :
3239 : /* For BB SLP there is no next iteration. */
3240 711464 : if (!loop_vinfo)
3241 576733 : return build_zero_cst (sizetype);
3242 :
3243 134731 : tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
3244 :
3245 : /* gather/scatter never reach here. */
3246 134731 : gcc_assert (!mat_gather_scatter_p (memory_access_type));
3247 :
3248 134731 : tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
3249 :
3250 134731 : return fold_build2 (MULT_EXPR, sizetype, iv_increment,
3251 : fold_convert (sizetype, step));
3252 : }
3253 :
3254 : /* Return the amount that should be added to a vector pointer to move
3255 : to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3256 : being vectorized and MEMORY_ACCESS_TYPE describes the type of
3257 : vectorization. */
3258 :
3259 : static tree
3260 711976 : vect_get_data_ptr_bump (vec_info *vinfo,
3261 : dr_vec_info *dr_info, tree aggr_type,
3262 : vect_memory_access_type memory_access_type)
3263 : {
3264 711976 : if (memory_access_type == VMAT_INVARIANT)
3265 0 : return size_zero_node;
3266 :
3267 711976 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3268 : /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
3269 : should never be needed. */
3270 135243 : if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
3271 : return NULL_TREE;
3272 :
3273 711976 : tree iv_step = TYPE_SIZE_UNIT (aggr_type);
3274 711976 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
3275 711976 : if (tree_int_cst_sgn (step) == -1)
3276 2840 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
3277 : return iv_step;
3278 : }
3279 :
3280 : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3281 :
3282 : static bool
3283 150 : vectorizable_bswap (vec_info *vinfo,
3284 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3285 : slp_tree slp_node,
3286 : slp_tree *slp_op,
3287 : tree vectype_in, stmt_vector_for_cost *cost_vec)
3288 : {
3289 150 : tree op, vectype;
3290 150 : gcall *stmt = as_a <gcall *> (stmt_info->stmt);
3291 :
3292 150 : op = gimple_call_arg (stmt, 0);
3293 150 : vectype = SLP_TREE_VECTYPE (slp_node);
3294 150 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3295 :
3296 150 : if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
3297 : {
3298 0 : if (dump_enabled_p ())
3299 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3300 : "mismatched vector sizes %T and %T\n",
3301 : vectype_in, vectype);
3302 0 : return false;
3303 : }
3304 :
3305 150 : tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
3306 150 : if (! char_vectype)
3307 : return false;
3308 :
3309 150 : poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
3310 150 : unsigned word_bytes;
3311 150 : if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
3312 : return false;
3313 :
3314 : /* The encoding uses one stepped pattern for each byte in the word. */
3315 150 : vec_perm_builder elts (num_bytes, word_bytes, 3);
3316 600 : for (unsigned i = 0; i < 3; ++i)
3317 2682 : for (unsigned j = 0; j < word_bytes; ++j)
3318 2232 : elts.quick_push ((i + 1) * word_bytes - j - 1);
3319 :
3320 150 : vec_perm_indices indices (elts, 1, num_bytes);
3321 150 : machine_mode vmode = TYPE_MODE (char_vectype);
3322 150 : if (!can_vec_perm_const_p (vmode, vmode, indices))
3323 : return false;
3324 :
3325 67 : if (cost_vec)
3326 : {
3327 51 : if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
3328 : {
3329 0 : if (dump_enabled_p ())
3330 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3331 : "incompatible vector types for invariants\n");
3332 0 : return false;
3333 : }
3334 :
3335 51 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3336 51 : DUMP_VECT_SCOPE ("vectorizable_bswap");
3337 51 : record_stmt_cost (cost_vec,
3338 : 1, vector_stmt, slp_node, 0, vect_prologue);
3339 51 : record_stmt_cost (cost_vec,
3340 51 : vect_get_num_copies (vinfo, slp_node),
3341 : vec_perm, slp_node, 0, vect_body);
3342 51 : return true;
3343 : }
3344 :
3345 16 : tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
3346 :
3347 : /* Transform. */
3348 16 : vec<tree> vec_oprnds = vNULL;
3349 16 : vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
3350 : /* Arguments are ready. create the new vector stmt. */
3351 16 : unsigned i;
3352 16 : tree vop;
3353 32 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
3354 : {
3355 16 : gimple *new_stmt;
3356 16 : tree tem = make_ssa_name (char_vectype);
3357 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3358 : char_vectype, vop));
3359 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3360 16 : tree tem2 = make_ssa_name (char_vectype);
3361 16 : new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
3362 : tem, tem, bswap_vconst);
3363 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3364 16 : tem = make_ssa_name (vectype);
3365 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3366 : vectype, tem2));
3367 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3368 16 : slp_node->push_vec_def (new_stmt);
3369 : }
3370 :
3371 16 : vec_oprnds.release ();
3372 16 : return true;
3373 150 : }
3374 :
3375 : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3376 : integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3377 : in a single step. On success, store the binary pack code in
3378 : *CONVERT_CODE. */
3379 :
3380 : static bool
3381 188 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
3382 : code_helper *convert_code)
3383 : {
3384 376 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
3385 376 : || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
3386 : return false;
3387 :
3388 78 : code_helper code;
3389 78 : int multi_step_cvt = 0;
3390 78 : auto_vec <tree, 8> interm_types;
3391 115 : if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
3392 : &code, &multi_step_cvt, &interm_types)
3393 78 : || multi_step_cvt)
3394 37 : return false;
3395 :
3396 41 : *convert_code = code;
3397 41 : return true;
3398 78 : }
3399 :
3400 : /* Function vectorizable_call.
3401 :
3402 : Check if STMT_INFO performs a function call that can be vectorized.
3403 : If COST_VEC is passed, calculate costs but don't change anything,
3404 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
3405 : it, and insert it at GSI.
3406 : Return true if STMT_INFO is vectorizable in this way. */
3407 :
3408 : static bool
3409 2718755 : vectorizable_call (vec_info *vinfo,
3410 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3411 : slp_tree slp_node,
3412 : stmt_vector_for_cost *cost_vec)
3413 : {
3414 2718755 : gcall *stmt;
3415 2718755 : tree vec_dest;
3416 2718755 : tree scalar_dest;
3417 2718755 : tree op;
3418 2718755 : tree vec_oprnd0 = NULL_TREE;
3419 2718755 : tree vectype_out, vectype_in;
3420 2718755 : poly_uint64 nunits_in;
3421 2718755 : poly_uint64 nunits_out;
3422 2718755 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3423 2718755 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3424 2718755 : tree fndecl, new_temp, rhs_type;
3425 2718755 : enum vect_def_type dt[5]
3426 : = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
3427 : vect_unknown_def_type, vect_unknown_def_type };
3428 2718755 : tree vectypes[ARRAY_SIZE (dt)] = {};
3429 2718755 : slp_tree slp_op[ARRAY_SIZE (dt)] = {};
3430 2718755 : auto_vec<tree, 8> vargs;
3431 2718755 : enum { NARROW, NONE, WIDEN } modifier;
3432 2718755 : size_t i, nargs;
3433 2718755 : tree clz_ctz_arg1 = NULL_TREE;
3434 :
3435 2718755 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3436 : return false;
3437 :
3438 2718755 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3439 237977 : && cost_vec)
3440 : return false;
3441 :
3442 : /* Is STMT_INFO a vectorizable call? */
3443 2733249 : stmt = dyn_cast <gcall *> (stmt_info->stmt);
3444 25654 : if (!stmt)
3445 : return false;
3446 :
3447 25654 : if (gimple_call_internal_p (stmt)
3448 25654 : && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3449 16459 : || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3450 : /* Handled by vectorizable_load and vectorizable_store. */
3451 3804 : return false;
3452 :
3453 21850 : if (gimple_call_lhs (stmt) == NULL_TREE
3454 21850 : || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3455 : return false;
3456 :
3457 21844 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
3458 :
3459 21844 : vectype_out = SLP_TREE_VECTYPE (slp_node);
3460 :
3461 : /* Process function arguments. */
3462 21844 : rhs_type = NULL_TREE;
3463 21844 : vectype_in = NULL_TREE;
3464 21844 : nargs = gimple_call_num_args (stmt);
3465 :
3466 : /* Bail out if the function has more than four arguments, we do not have
3467 : interesting builtin functions to vectorize with more than two arguments
3468 : except for fma (cond_fma has more). No arguments is also not good. */
3469 21844 : if (nargs == 0 || nargs > 5)
3470 : return false;
3471 :
3472 : /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3473 21764 : combined_fn cfn = gimple_call_combined_fn (stmt);
3474 21764 : if (cfn == CFN_GOMP_SIMD_LANE)
3475 : {
3476 3199 : nargs = 0;
3477 3199 : rhs_type = unsigned_type_node;
3478 : }
3479 : /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3480 : argument just says whether it is well-defined at zero or not and what
3481 : value should be returned for it. */
3482 21764 : if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
3483 : {
3484 168 : nargs = 1;
3485 168 : clz_ctz_arg1 = gimple_call_arg (stmt, 1);
3486 : }
3487 :
3488 21764 : int mask_opno = -1;
3489 21764 : if (internal_fn_p (cfn))
3490 : {
3491 : /* We can only handle direct internal masked calls here,
3492 : vectorizable_simd_clone_call is for the rest. */
3493 18696 : if (cfn == CFN_MASK_CALL)
3494 : return false;
3495 18542 : mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
3496 : }
3497 :
3498 68075 : for (i = 0; i < nargs; i++)
3499 : {
3500 48477 : if ((int) i == mask_opno)
3501 : {
3502 7694 : if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
3503 : &slp_op[i], &dt[i], &vectypes[i]))
3504 : return false;
3505 7694 : continue;
3506 : }
3507 :
3508 40783 : if (!vect_is_simple_use (vinfo, slp_node,
3509 : i, &op, &slp_op[i], &dt[i], &vectypes[i]))
3510 : {
3511 0 : if (dump_enabled_p ())
3512 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3513 : "use not simple.\n");
3514 0 : return false;
3515 : }
3516 :
3517 : /* We can only handle calls with arguments of the same type. */
3518 40783 : if (rhs_type
3519 40783 : && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3520 : {
3521 2012 : if (dump_enabled_p ())
3522 200 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3523 : "argument types differ.\n");
3524 2012 : return false;
3525 : }
3526 38771 : if (!rhs_type)
3527 18411 : rhs_type = TREE_TYPE (op);
3528 :
3529 38771 : if (!vectype_in)
3530 19667 : vectype_in = vectypes[i];
3531 19104 : else if (vectypes[i]
3532 19104 : && !types_compatible_p (vectypes[i], vectype_in))
3533 : {
3534 0 : if (dump_enabled_p ())
3535 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3536 : "argument vector types differ.\n");
3537 0 : return false;
3538 : }
3539 : }
3540 : /* If all arguments are external or constant defs, infer the vector type
3541 : from the scalar type. */
3542 19598 : if (!vectype_in)
3543 5525 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
3544 19598 : if (!cost_vec)
3545 4193 : gcc_assert (vectype_in);
3546 15405 : if (!vectype_in)
3547 : {
3548 1041 : if (dump_enabled_p ())
3549 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3550 : "no vectype for scalar type %T\n", rhs_type);
3551 :
3552 1041 : return false;
3553 : }
3554 :
3555 37114 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3556 18557 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
3557 : {
3558 12 : if (dump_enabled_p ())
3559 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3560 : "mixed mask and nonmask vector types\n");
3561 12 : return false;
3562 : }
3563 :
3564 18545 : if (vect_emulated_vector_p (vectype_in)
3565 18545 : || vect_emulated_vector_p (vectype_out))
3566 : {
3567 0 : if (dump_enabled_p ())
3568 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3569 : "use emulated vector type for call\n");
3570 0 : return false;
3571 : }
3572 :
3573 : /* FORNOW */
3574 18545 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3575 18545 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3576 18545 : if (known_eq (nunits_in * 2, nunits_out))
3577 : modifier = NARROW;
3578 17970 : else if (known_eq (nunits_out, nunits_in))
3579 : modifier = NONE;
3580 45 : else if (known_eq (nunits_out * 2, nunits_in))
3581 : modifier = WIDEN;
3582 : else
3583 : return false;
3584 :
3585 : /* We only handle functions that do not read or clobber memory. */
3586 37090 : if (gimple_vuse (stmt))
3587 : {
3588 1231 : if (dump_enabled_p ())
3589 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3590 : "function reads from or writes to memory.\n");
3591 1231 : return false;
3592 : }
3593 :
3594 : /* For now, we only vectorize functions if a target specific builtin
3595 : is available. TODO -- in some cases, it might be profitable to
3596 : insert the calls for pieces of the vector, in order to be able
3597 : to vectorize other operations in the loop. */
3598 17314 : fndecl = NULL_TREE;
3599 17314 : internal_fn ifn = IFN_LAST;
3600 17314 : tree callee = gimple_call_fndecl (stmt);
3601 :
3602 : /* First try using an internal function. */
3603 17314 : code_helper convert_code = MAX_TREE_CODES;
3604 17314 : if (cfn != CFN_LAST
3605 17314 : && (modifier == NONE
3606 200 : || (modifier == NARROW
3607 188 : && simple_integer_narrowing (vectype_out, vectype_in,
3608 : &convert_code))))
3609 16285 : ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3610 : vectype_in);
3611 :
3612 : /* Check if the operation traps. */
3613 17314 : bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
3614 17314 : if (could_trap && cost_vec && loop_vinfo)
3615 : {
3616 : /* If the operation can trap it must be conditional, otherwise fail. */
3617 474 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3618 474 : ? ifn : get_conditional_internal_fn (ifn));
3619 474 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3620 474 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
3621 : {
3622 : /* We assume that BB SLP fills all lanes, so no inactive lanes can
3623 : cause issues. */
3624 84 : if ((cond_fn == IFN_LAST
3625 56 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3626 : OPTIMIZE_FOR_SPEED))
3627 140 : && (cond_len_fn == IFN_LAST
3628 56 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3629 : OPTIMIZE_FOR_SPEED)))
3630 : {
3631 84 : if (dump_enabled_p ())
3632 10 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3633 : "can't use a fully-masked loop because no"
3634 : " conditional operation is available.\n");
3635 84 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3636 : }
3637 : }
3638 : }
3639 :
3640 : /* If that fails, try asking for a target-specific built-in function. */
3641 17314 : if (ifn == IFN_LAST)
3642 : {
3643 9883 : if (cfn != CFN_LAST)
3644 9013 : fndecl = targetm.vectorize.builtin_vectorized_function
3645 9013 : (cfn, vectype_out, vectype_in);
3646 870 : else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
3647 24 : fndecl = targetm.vectorize.builtin_md_vectorized_function
3648 24 : (callee, vectype_out, vectype_in);
3649 : }
3650 :
3651 17314 : if (ifn == IFN_LAST && !fndecl)
3652 : {
3653 9503 : if (cfn == CFN_GOMP_SIMD_LANE
3654 3199 : && SLP_TREE_LANES (slp_node) == 1
3655 3199 : && loop_vinfo
3656 3199 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3657 3199 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3658 15901 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3659 3199 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3660 : {
3661 : /* We can handle IFN_GOMP_SIMD_LANE by returning a
3662 : { 0, 1, 2, ... vf - 1 } vector. */
3663 3199 : gcc_assert (nargs == 0);
3664 : }
3665 6304 : else if (modifier == NONE
3666 6304 : && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3667 5954 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3668 5891 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
3669 5838 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
3670 150 : return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
3671 150 : slp_op, vectype_in, cost_vec);
3672 : else
3673 : {
3674 6154 : if (dump_enabled_p ())
3675 262 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3676 : "function is not vectorizable.\n");
3677 6154 : return false;
3678 : }
3679 : }
3680 :
3681 11010 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
3682 11010 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3683 11010 : ? ifn : get_conditional_internal_fn (ifn));
3684 11010 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3685 11010 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3686 9150 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
3687 11010 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
3688 11010 : if (cost_vec) /* transformation not required. */
3689 : {
3690 21681 : for (i = 0; i < nargs; ++i)
3691 14848 : if (!vect_maybe_update_slp_op_vectype (slp_op[i],
3692 14848 : vectypes[i]
3693 : ? vectypes[i] : vectype_in))
3694 : {
3695 0 : if (dump_enabled_p ())
3696 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3697 : "incompatible vector types for invariants\n");
3698 0 : return false;
3699 : }
3700 6833 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3701 6833 : DUMP_VECT_SCOPE ("vectorizable_call");
3702 6833 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
3703 :
3704 6833 : if (loop_vinfo
3705 5899 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
3706 4056 : && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3707 : {
3708 2558 : if (reduc_idx >= 0
3709 1631 : && (cond_fn == IFN_LAST
3710 1631 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3711 : OPTIMIZE_FOR_SPEED))
3712 2570 : && (cond_len_fn == IFN_LAST
3713 12 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3714 : OPTIMIZE_FOR_SPEED)))
3715 : {
3716 12 : if (dump_enabled_p ())
3717 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3718 : "can't use a fully-masked loop because no"
3719 : " conditional operation is available.\n");
3720 12 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3721 : }
3722 : else
3723 : {
3724 2546 : tree scalar_mask = NULL_TREE;
3725 2546 : if (mask_opno >= 0)
3726 2546 : scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3727 2546 : if (cond_len_fn != IFN_LAST
3728 2546 : && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3729 : OPTIMIZE_FOR_SPEED))
3730 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3731 : 1);
3732 : else
3733 2546 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3734 : scalar_mask);
3735 : }
3736 : }
3737 6833 : return true;
3738 : }
3739 :
3740 : /* Transform. */
3741 :
3742 4177 : if (dump_enabled_p ())
3743 416 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3744 :
3745 : /* Handle def. */
3746 4177 : scalar_dest = gimple_call_lhs (stmt);
3747 4177 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3748 :
3749 4177 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3750 3251 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
3751 4177 : unsigned int vect_nargs = nargs;
3752 4177 : if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3753 : {
3754 0 : ifn = cond_len_fn;
3755 : /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3756 0 : vect_nargs += 2;
3757 : /* But unless there's a mask argument already we need that
3758 : as well, and an else value. */
3759 0 : if (mask_opno == -1)
3760 0 : vect_nargs += 2;
3761 : }
3762 4177 : else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
3763 : {
3764 0 : ifn = cond_fn;
3765 0 : vect_nargs += 2;
3766 : }
3767 4177 : int len_opno = internal_fn_len_index (ifn);
3768 4177 : if (clz_ctz_arg1)
3769 59 : ++vect_nargs;
3770 :
3771 4177 : if (modifier == NONE || ifn != IFN_LAST)
3772 : {
3773 4145 : tree prev_res = NULL_TREE;
3774 4145 : vargs.safe_grow (vect_nargs, true);
3775 4145 : auto_vec<vec<tree> > vec_defs (nargs);
3776 :
3777 : /* Build argument list for the vectorized call. */
3778 4145 : if (cfn == CFN_GOMP_SIMD_LANE)
3779 : {
3780 3300 : for (i = 0; i < nvectors; ++i)
3781 : {
3782 : /* ??? For multi-lane SLP we'd need to build
3783 : { 0, 0, .., 1, 1, ... }. */
3784 1704 : tree cst = build_index_vector (vectype_out,
3785 : i * nunits_out, 1);
3786 1704 : tree new_var
3787 1704 : = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3788 1704 : gimple *init_stmt = gimple_build_assign (new_var, cst);
3789 1704 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
3790 1704 : new_temp = make_ssa_name (vec_dest);
3791 1704 : gimple *new_stmt = gimple_build_assign (new_temp, new_var);
3792 1704 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3793 1704 : slp_node->push_vec_def (new_stmt);
3794 : }
3795 : }
3796 : else
3797 : {
3798 2549 : vec<tree> vec_oprnds0;
3799 2549 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3800 2549 : vec_oprnds0 = vec_defs[0];
3801 :
3802 : /* Arguments are ready. Create the new vector stmt. */
3803 5251 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3804 : {
3805 2702 : int varg = 0;
3806 : /* Add the mask if necessary. */
3807 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3808 2704 : && internal_fn_mask_index (ifn) != -1)
3809 : {
3810 0 : gcc_assert (internal_fn_mask_index (ifn) == varg);
3811 0 : if (masked_loop_p)
3812 : {
3813 0 : unsigned int vec_num = vec_oprnds0.length ();
3814 0 : vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
3815 : masks, vec_num,
3816 : vectype_out, i);
3817 : }
3818 : else
3819 : {
3820 0 : tree mask_vectype = truth_type_for (vectype_out);
3821 0 : vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
3822 : stmt_info,
3823 : mask_vectype);
3824 : }
3825 : }
3826 : size_t k;
3827 9911 : for (k = 0; k < nargs; k++)
3828 : {
3829 7209 : vec<tree> vec_oprndsk = vec_defs[k];
3830 7209 : vargs[varg++] = vec_oprndsk[i];
3831 : }
3832 : /* Add the else value if necessary. */
3833 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3834 2704 : && internal_fn_else_index (ifn) != -1)
3835 : {
3836 0 : gcc_assert (internal_fn_else_index (ifn) == varg);
3837 0 : if (reduc_idx >= 0)
3838 0 : vargs[varg++] = vargs[reduc_idx + 1];
3839 : else
3840 : {
3841 0 : auto else_value = targetm.preferred_else_value
3842 0 : (ifn, vectype_out, varg - 1, &vargs[1]);
3843 0 : vargs[varg++] = else_value;
3844 : }
3845 : }
3846 2702 : if (clz_ctz_arg1)
3847 59 : vargs[varg++] = clz_ctz_arg1;
3848 :
3849 2702 : gimple *new_stmt;
3850 2702 : if (modifier == NARROW)
3851 : {
3852 : /* We don't define any narrowing conditional functions
3853 : at present. */
3854 0 : gcc_assert (mask_opno < 0);
3855 0 : tree half_res = make_ssa_name (vectype_in);
3856 0 : gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3857 0 : gimple_call_set_lhs (call, half_res);
3858 0 : gimple_call_set_nothrow (call, true);
3859 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3860 0 : if ((i & 1) == 0)
3861 : {
3862 0 : prev_res = half_res;
3863 0 : continue;
3864 : }
3865 0 : new_temp = make_ssa_name (vec_dest);
3866 0 : new_stmt = vect_gimple_build (new_temp, convert_code,
3867 : prev_res, half_res);
3868 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3869 : }
3870 : else
3871 : {
3872 2702 : if (len_opno >= 0 && len_loop_p)
3873 : {
3874 0 : unsigned int vec_num = vec_oprnds0.length ();
3875 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
3876 : vec_num, vectype_out, i, 1, true);
3877 0 : signed char biasval
3878 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3879 0 : tree bias = build_int_cst (intQI_type_node, biasval);
3880 0 : vargs[len_opno] = len;
3881 0 : vargs[len_opno + 1] = bias;
3882 : }
3883 2702 : else if (mask_opno >= 0 && masked_loop_p)
3884 : {
3885 36 : unsigned int vec_num = vec_oprnds0.length ();
3886 36 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
3887 : vec_num, vectype_out, i);
3888 36 : vargs[mask_opno]
3889 72 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
3890 36 : vargs[mask_opno], gsi);
3891 : }
3892 :
3893 2702 : gcall *call;
3894 2702 : if (ifn != IFN_LAST)
3895 2621 : call = gimple_build_call_internal_vec (ifn, vargs);
3896 : else
3897 81 : call = gimple_build_call_vec (fndecl, vargs);
3898 2702 : new_temp = make_ssa_name (vec_dest, call);
3899 2702 : gimple_call_set_lhs (call, new_temp);
3900 2702 : gimple_call_set_nothrow (call, true);
3901 2702 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3902 2702 : new_stmt = call;
3903 : }
3904 2702 : slp_node->push_vec_def (new_stmt);
3905 : }
3906 : }
3907 :
3908 10992 : for (i = 0; i < nargs; i++)
3909 : {
3910 6847 : vec<tree> vec_oprndsi = vec_defs[i];
3911 6847 : vec_oprndsi.release ();
3912 : }
3913 4145 : }
3914 32 : else if (modifier == NARROW)
3915 : {
3916 32 : auto_vec<vec<tree> > vec_defs (nargs);
3917 : /* We don't define any narrowing conditional functions at present. */
3918 32 : gcc_assert (mask_opno < 0);
3919 :
3920 : /* Build argument list for the vectorized call. */
3921 32 : vargs.create (nargs * 2);
3922 :
3923 32 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3924 32 : vec<tree> vec_oprnds0 = vec_defs[0];
3925 :
3926 : /* Arguments are ready. Create the new vector stmt. */
3927 64 : for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3928 : {
3929 32 : size_t k;
3930 32 : vargs.truncate (0);
3931 64 : for (k = 0; k < nargs; k++)
3932 : {
3933 32 : vec<tree> vec_oprndsk = vec_defs[k];
3934 32 : vargs.quick_push (vec_oprndsk[i]);
3935 32 : vargs.quick_push (vec_oprndsk[i + 1]);
3936 : }
3937 32 : gcall *call;
3938 32 : if (ifn != IFN_LAST)
3939 : call = gimple_build_call_internal_vec (ifn, vargs);
3940 : else
3941 32 : call = gimple_build_call_vec (fndecl, vargs);
3942 32 : new_temp = make_ssa_name (vec_dest, call);
3943 32 : gimple_call_set_lhs (call, new_temp);
3944 32 : gimple_call_set_nothrow (call, true);
3945 32 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3946 32 : slp_node->push_vec_def (call);
3947 : }
3948 :
3949 64 : for (i = 0; i < nargs; i++)
3950 : {
3951 32 : vec<tree> vec_oprndsi = vec_defs[i];
3952 32 : vec_oprndsi.release ();
3953 : }
3954 32 : }
3955 : else
3956 : /* No current target implements this case. */
3957 : return false;
3958 :
3959 4177 : vargs.release ();
3960 :
3961 4177 : return true;
3962 2718755 : }
3963 :
3964 :
3965 : struct simd_call_arg_info
3966 : {
3967 : tree vectype;
3968 : tree op;
3969 : HOST_WIDE_INT linear_step;
3970 : enum vect_def_type dt;
3971 : unsigned int align;
3972 : bool simd_lane_linear;
3973 : };
3974 :
3975 : /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3976 : is linear within simd lane (but not within whole loop), note it in
3977 : *ARGINFO. */
3978 :
3979 : static void
3980 15 : vect_simd_lane_linear (tree op, class loop *loop,
3981 : struct simd_call_arg_info *arginfo)
3982 : {
3983 15 : gimple *def_stmt = SSA_NAME_DEF_STMT (op);
3984 :
3985 15 : if (!is_gimple_assign (def_stmt)
3986 15 : || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
3987 27 : || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
3988 3 : return;
3989 :
3990 12 : tree base = gimple_assign_rhs1 (def_stmt);
3991 12 : HOST_WIDE_INT linear_step = 0;
3992 12 : tree v = gimple_assign_rhs2 (def_stmt);
3993 48 : while (TREE_CODE (v) == SSA_NAME)
3994 : {
3995 36 : tree t;
3996 36 : def_stmt = SSA_NAME_DEF_STMT (v);
3997 36 : if (is_gimple_assign (def_stmt))
3998 24 : switch (gimple_assign_rhs_code (def_stmt))
3999 : {
4000 0 : case PLUS_EXPR:
4001 0 : t = gimple_assign_rhs2 (def_stmt);
4002 0 : if (linear_step || TREE_CODE (t) != INTEGER_CST)
4003 : return;
4004 0 : base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
4005 0 : v = gimple_assign_rhs1 (def_stmt);
4006 0 : continue;
4007 12 : case MULT_EXPR:
4008 12 : t = gimple_assign_rhs2 (def_stmt);
4009 12 : if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
4010 0 : return;
4011 12 : linear_step = tree_to_shwi (t);
4012 12 : v = gimple_assign_rhs1 (def_stmt);
4013 12 : continue;
4014 12 : CASE_CONVERT:
4015 12 : t = gimple_assign_rhs1 (def_stmt);
4016 12 : if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
4017 12 : || (TYPE_PRECISION (TREE_TYPE (v))
4018 12 : < TYPE_PRECISION (TREE_TYPE (t))))
4019 : return;
4020 12 : if (!linear_step)
4021 0 : linear_step = 1;
4022 12 : v = t;
4023 12 : continue;
4024 : default:
4025 : return;
4026 : }
4027 12 : else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
4028 12 : && loop->simduid
4029 12 : && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
4030 24 : && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
4031 : == loop->simduid))
4032 : {
4033 12 : if (!linear_step)
4034 0 : linear_step = 1;
4035 12 : arginfo->linear_step = linear_step;
4036 12 : arginfo->op = base;
4037 12 : arginfo->simd_lane_linear = true;
4038 12 : return;
4039 : }
4040 : }
4041 : }
4042 :
4043 : /* Function vectorizable_simd_clone_call.
4044 :
4045 : Check if STMT_INFO performs a function call that can be vectorized
4046 : by calling a simd clone of the function.
4047 : If COST_VEC is passed, calculate costs but don't change anything,
4048 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
4049 : it, and insert it at GSI.
4050 : Return true if STMT_INFO is vectorizable in this way. */
4051 :
4052 : static bool
4053 2708036 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
4054 : gimple_stmt_iterator *gsi,
4055 : slp_tree slp_node,
4056 : stmt_vector_for_cost *cost_vec)
4057 : {
4058 2708036 : tree vec_dest;
4059 2708036 : tree scalar_dest;
4060 2708036 : tree vec_oprnd0 = NULL_TREE;
4061 2708036 : tree vectype;
4062 2708036 : poly_uint64 nunits;
4063 2708036 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
4064 2708036 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
4065 2708036 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
4066 2708036 : tree fndecl, new_temp;
4067 2708036 : int j;
4068 2708036 : auto_vec<simd_call_arg_info> arginfo;
4069 2708036 : vec<tree> vargs = vNULL;
4070 2708036 : size_t i, nargs;
4071 2708036 : tree rtype, ratype;
4072 2708036 : vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
4073 2708036 : int masked_call_offset = 0;
4074 :
4075 : /* Is STMT a vectorizable call? */
4076 2708036 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
4077 16124 : if (!stmt)
4078 : return false;
4079 :
4080 16124 : fndecl = gimple_call_fndecl (stmt);
4081 16124 : if (fndecl == NULL_TREE
4082 16124 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
4083 : {
4084 220 : fndecl = gimple_call_arg (stmt, 0);
4085 220 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
4086 220 : fndecl = TREE_OPERAND (fndecl, 0);
4087 220 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
4088 : masked_call_offset = 1;
4089 : }
4090 15904 : if (fndecl == NULL_TREE)
4091 : return false;
4092 :
4093 5690 : struct cgraph_node *node = cgraph_node::get (fndecl);
4094 5690 : if (node == NULL || node->simd_clones == NULL)
4095 : return false;
4096 :
4097 1460 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4098 : return false;
4099 :
4100 1460 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4101 0 : && cost_vec)
4102 : return false;
4103 :
4104 1460 : if (gimple_call_lhs (stmt)
4105 1460 : && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
4106 : return false;
4107 :
4108 1460 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
4109 :
4110 1460 : vectype = SLP_TREE_VECTYPE (slp_node);
4111 :
4112 2708100 : if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
4113 : return false;
4114 :
4115 : /* Process function arguments. */
4116 1460 : nargs = gimple_call_num_args (stmt) - masked_call_offset;
4117 :
4118 : /* Bail out if the function has zero arguments. */
4119 1460 : if (nargs == 0)
4120 : return false;
4121 :
4122 1396 : vect_simd_clone_data _data;
4123 1396 : vect_simd_clone_data &data = slp_node->get_data (_data);
4124 1396 : vec<tree>& simd_clone_info = data.simd_clone_info;
4125 1396 : arginfo.reserve (nargs, true);
4126 1396 : auto_vec<slp_tree> slp_op;
4127 1396 : slp_op.safe_grow_cleared (nargs);
4128 :
4129 4021 : for (i = 0; i < nargs; i++)
4130 : {
4131 2625 : simd_call_arg_info thisarginfo;
4132 2625 : affine_iv iv;
4133 2625 : tree op;
4134 :
4135 2625 : thisarginfo.linear_step = 0;
4136 2625 : thisarginfo.align = 0;
4137 2625 : thisarginfo.op = NULL_TREE;
4138 2625 : thisarginfo.simd_lane_linear = false;
4139 :
4140 5250 : int op_no = vect_slp_child_index_for_operand (stmt_info,
4141 2625 : i + masked_call_offset);
4142 5250 : if (!vect_is_simple_use (vinfo, slp_node,
4143 2625 : op_no, &op, &slp_op[i],
4144 : &thisarginfo.dt, &thisarginfo.vectype)
4145 2625 : || thisarginfo.dt == vect_uninitialized_def)
4146 : {
4147 0 : if (dump_enabled_p ())
4148 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4149 : "use not simple.\n");
4150 0 : return false;
4151 : }
4152 :
4153 2625 : if (thisarginfo.dt == vect_constant_def
4154 2625 : || thisarginfo.dt == vect_external_def)
4155 : {
4156 : /* With SLP we determine the vector type of constants/externals
4157 : at analysis time, handling conflicts via
4158 : vect_maybe_update_slp_op_vectype. At transform time
4159 : we have a vector type recorded for SLP. */
4160 680 : gcc_assert (cost_vec
4161 : || thisarginfo.vectype != NULL_TREE);
4162 : if (cost_vec)
4163 549 : thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
4164 549 : TREE_TYPE (op),
4165 : slp_node);
4166 : }
4167 : else
4168 1945 : gcc_assert (thisarginfo.vectype != NULL_TREE);
4169 :
4170 : /* For linear arguments, the analyze phase should have saved
4171 : the base and step. */
4172 2494 : if (!cost_vec
4173 1586 : && i * 3 + 4 <= simd_clone_info.length ()
4174 2704 : && simd_clone_info[i * 3 + 2])
4175 : {
4176 118 : thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
4177 118 : thisarginfo.op = simd_clone_info[i * 3 + 1];
4178 118 : thisarginfo.simd_lane_linear
4179 118 : = (simd_clone_info[i * 3 + 3] == boolean_true_node);
4180 : /* If loop has been peeled for alignment, we need to adjust it. */
4181 118 : tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
4182 118 : tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
4183 118 : if (n1 != n2 && !thisarginfo.simd_lane_linear)
4184 : {
4185 0 : tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
4186 0 : tree step = simd_clone_info[i * 3 + 2];
4187 0 : tree opt = TREE_TYPE (thisarginfo.op);
4188 0 : bias = fold_convert (TREE_TYPE (step), bias);
4189 0 : bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
4190 0 : thisarginfo.op
4191 0 : = fold_build2 (POINTER_TYPE_P (opt)
4192 : ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
4193 : thisarginfo.op, bias);
4194 : }
4195 : }
4196 2507 : else if (cost_vec
4197 1832 : && thisarginfo.dt != vect_constant_def
4198 1705 : && thisarginfo.dt != vect_external_def
4199 1283 : && loop_vinfo
4200 1278 : && SLP_TREE_LANES (slp_node) == 1
4201 1254 : && TREE_CODE (op) == SSA_NAME
4202 2508 : && simple_iv (loop, loop_containing_stmt (stmt), op,
4203 : &iv, false)
4204 2719 : && tree_fits_shwi_p (iv.step))
4205 : {
4206 212 : thisarginfo.linear_step = tree_to_shwi (iv.step);
4207 212 : thisarginfo.op = iv.base;
4208 : }
4209 2295 : else if ((thisarginfo.dt == vect_constant_def
4210 2295 : || thisarginfo.dt == vect_external_def)
4211 680 : && SLP_TREE_LANES (slp_node) == 1
4212 2601 : && POINTER_TYPE_P (TREE_TYPE (op)))
4213 86 : thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
4214 : /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4215 : linear too. */
4216 2625 : if (SLP_TREE_LANES (slp_node) == 1
4217 2205 : && POINTER_TYPE_P (TREE_TYPE (op))
4218 196 : && !thisarginfo.linear_step
4219 112 : && cost_vec
4220 58 : && thisarginfo.dt != vect_constant_def
4221 58 : && thisarginfo.dt != vect_external_def
4222 15 : && loop_vinfo
4223 2640 : && TREE_CODE (op) == SSA_NAME)
4224 15 : vect_simd_lane_linear (op, loop, &thisarginfo);
4225 :
4226 2625 : if (!vectype)
4227 12 : vectype = thisarginfo.vectype;
4228 2625 : arginfo.quick_push (thisarginfo);
4229 : }
4230 :
4231 1396 : poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
4232 1396 : unsigned group_size = SLP_TREE_LANES (slp_node);
4233 1396 : unsigned int badness = 0;
4234 1396 : unsigned int badness_inbranch = 0;
4235 1396 : struct cgraph_node *bestn = NULL;
4236 1396 : struct cgraph_node *bestn_inbranch = NULL;
4237 1396 : if (!cost_vec)
4238 358 : bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4239 358 : ? data.clone_inbranch : data.clone);
4240 : else
4241 5968 : for (struct cgraph_node *n = node->simd_clones; n != NULL;
4242 4930 : n = n->simdclone->next_clone)
4243 : {
4244 4930 : unsigned int this_badness = 0;
4245 4930 : unsigned int num_calls;
4246 : /* The number of arguments in the call and the number of parameters in
4247 : the simdclone should match. However, when the simdclone is
4248 : 'inbranch', it could have one more parameter than nargs when using
4249 : an inbranch simdclone to call a non-inbranch call, either in a
4250 : non-masked loop using a all true constant mask, or inside a masked
4251 : loop using it's mask. */
4252 4930 : size_t simd_nargs = n->simdclone->nargs;
4253 4930 : if (!masked_call_offset && n->simdclone->inbranch)
4254 2223 : simd_nargs--;
4255 4930 : if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
4256 : &num_calls)
4257 1890 : || (!n->simdclone->inbranch && (masked_call_offset > 0))
4258 1706 : || (nargs != simd_nargs))
4259 3224 : continue;
4260 1706 : if (num_calls != 1)
4261 1046 : this_badness += floor_log2 (num_calls) * 4096;
4262 1706 : if (n->simdclone->inbranch)
4263 729 : this_badness += 8192;
4264 :
4265 : /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
4266 : mode, which for targets that use it will determine what ISA we can
4267 : vectorize this code with. */
4268 1706 : machine_mode vector_mode = vinfo->vector_mode;
4269 1706 : if (vectype)
4270 1706 : vector_mode = TYPE_MODE (vectype);
4271 1706 : int target_badness = targetm.simd_clone.usable (n, vector_mode);
4272 1706 : if (target_badness < 0)
4273 356 : continue;
4274 1350 : this_badness += target_badness * 512;
4275 4048 : for (i = 0; i < nargs; i++)
4276 : {
4277 2946 : switch (n->simdclone->args[i].arg_type)
4278 : {
4279 2016 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4280 2016 : if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
4281 : /* Vector mask arguments are not supported. */
4282 : i = -1;
4283 2008 : else if (!useless_type_conversion_p
4284 2008 : (n->simdclone->args[i].orig_type,
4285 2008 : TREE_TYPE (gimple_call_arg (stmt,
4286 : i + masked_call_offset))))
4287 : i = -1;
4288 2008 : else if (arginfo[i].dt == vect_constant_def
4289 1901 : || arginfo[i].dt == vect_external_def
4290 3845 : || arginfo[i].linear_step)
4291 399 : this_badness += 64;
4292 : break;
4293 310 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4294 310 : if ((arginfo[i].dt != vect_constant_def
4295 145 : && arginfo[i].dt != vect_external_def)
4296 410 : || SLP_TREE_LANES (slp_node) != 1)
4297 : i = -1;
4298 : break;
4299 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4300 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4301 324 : if (arginfo[i].dt == vect_constant_def
4302 324 : || arginfo[i].dt == vect_external_def
4303 324 : || (arginfo[i].linear_step
4304 324 : != n->simdclone->args[i].linear_step))
4305 : i = -1;
4306 : break;
4307 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4308 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4309 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4310 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4311 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4312 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4313 : /* FORNOW */
4314 : i = -1;
4315 : break;
4316 296 : case SIMD_CLONE_ARG_TYPE_MASK:
4317 296 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4318 264 : && n->simdclone->mask_mode != VOIDmode)
4319 : i = -1;
4320 : /* While we can create a traditional data vector from
4321 : an incoming integer mode mask we have no good way to
4322 : force generate an integer mode mask from a traditional
4323 : boolean vector input. */
4324 296 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4325 296 : && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4326 : i = -1;
4327 290 : else if (n->simdclone->mask_mode == VOIDmode
4328 : /* FORNOW we only have partial support for vector-type
4329 : masks that can't hold all of simdlen. */
4330 554 : && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
4331 264 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
4332 : /* Verify we can compute the mask argument. */
4333 111 : || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
4334 111 : arginfo[i].vectype)))
4335 : i = -1;
4336 125 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4337 : /* FORNOW we only have partial support for
4338 : integer-type masks that represent the same number
4339 : of lanes as the vectorized mask inputs. */
4340 151 : && maybe_ne (exact_div (n->simdclone->simdlen,
4341 : n->simdclone->args[i].linear_step),
4342 26 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4343 : i = -1;
4344 107 : else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4345 107 : && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4346 8 : this_badness += 2048;
4347 : break;
4348 : }
4349 183 : if (i == (size_t) -1)
4350 : break;
4351 2698 : if (n->simdclone->args[i].alignment > arginfo[i].align)
4352 : {
4353 : i = -1;
4354 : break;
4355 : }
4356 2698 : if (arginfo[i].align)
4357 110 : this_badness += (exact_log2 (arginfo[i].align)
4358 160 : - exact_log2 (n->simdclone->args[i].alignment));
4359 : }
4360 1350 : if (i == (size_t) -1)
4361 248 : continue;
4362 1102 : if (masked_call_offset == 0
4363 995 : && n->simdclone->inbranch
4364 311 : && n->simdclone->nargs > nargs)
4365 : {
4366 311 : gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
4367 : SIMD_CLONE_ARG_TYPE_MASK);
4368 : /* Penalize using a masked SIMD clone in a non-masked loop, that is
4369 : not in a branch, as we'd have to construct an all-true mask. */
4370 311 : this_badness += 64;
4371 : }
4372 1102 : if (bestn == NULL || this_badness < badness)
4373 : {
4374 781 : bestn = n;
4375 781 : badness = this_badness;
4376 : }
4377 1102 : if (n->simdclone->inbranch
4378 418 : && (bestn_inbranch == NULL || this_badness < badness_inbranch))
4379 : {
4380 4930 : bestn_inbranch = n;
4381 4930 : badness_inbranch = this_badness;
4382 : }
4383 : }
4384 :
4385 1396 : if (bestn == NULL)
4386 : return false;
4387 :
4388 813 : fndecl = bestn->decl;
4389 813 : nunits = bestn->simdclone->simdlen;
4390 813 : int ncopies = vector_unroll_factor (vf * group_size, nunits);
4391 :
4392 : /* If the function isn't const, only allow it in simd loops where user
4393 : has asserted that at least nunits consecutive iterations can be
4394 : performed using SIMD instructions. */
4395 808 : if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
4396 974 : && gimple_vuse (stmt))
4397 : return false;
4398 :
4399 : /* ncopies is the number of SIMD clone calls we create, since simdlen
4400 : is not necessarily matching nunits of the vector types used, track
4401 : that in ncopies_in. */
4402 813 : int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
4403 :
4404 : /* Sanity check: make sure that at least one copy of the vectorized stmt
4405 : needs to be generated. */
4406 813 : gcc_assert (ncopies >= 1);
4407 :
4408 813 : if (cost_vec) /* transformation not required. */
4409 : {
4410 1490 : for (unsigned i = 0; i < nargs; ++i)
4411 1035 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
4412 : {
4413 0 : if (dump_enabled_p ())
4414 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4415 : "incompatible vector types for invariants\n");
4416 0 : return false;
4417 : }
4418 :
4419 455 : if (!bestn_inbranch && loop_vinfo)
4420 : {
4421 248 : if (dump_enabled_p ()
4422 248 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
4423 171 : dump_printf_loc (MSG_NOTE, vect_location,
4424 : "can't use a fully-masked loop because no"
4425 : " masked simd clone was available.\n");
4426 248 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4427 : }
4428 :
4429 : /* When the original call is pure or const but the SIMD ABI dictates
4430 : an aggregate return we will have to use a virtual definition and
4431 : in a loop eventually even need to add a virtual PHI. That's
4432 : not straight-forward so allow to fix this up via renaming. */
4433 455 : if (gimple_call_lhs (stmt)
4434 449 : && !gimple_vdef (stmt)
4435 808 : && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
4436 27 : vinfo->any_known_not_updated_vssa = true;
4437 : /* ??? For SLP code-gen we end up inserting after the last
4438 : vector argument def rather than at the original call position
4439 : so automagic virtual operand updating doesn't work. */
4440 910 : if (gimple_vuse (stmt))
4441 139 : vinfo->any_known_not_updated_vssa = true;
4442 :
4443 455 : data.clone = bestn;
4444 455 : data.clone_inbranch = bestn_inbranch;
4445 :
4446 455 : simd_clone_info.safe_push (NULL_TREE);
4447 1627 : for (i = 0;
4448 2466 : i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
4449 : {
4450 1172 : if (loop_vinfo
4451 1166 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
4452 458 : && (bestn_inbranch->simdclone->args[i].arg_type
4453 : == SIMD_CLONE_ARG_TYPE_MASK))
4454 : {
4455 162 : if (masked_call_offset)
4456 : /* When there is an explicit mask we require the
4457 : number of elements to match up. */
4458 49 : vect_record_loop_mask (loop_vinfo,
4459 : &LOOP_VINFO_MASKS (loop_vinfo),
4460 : ncopies_in, vectype, NULL_TREE);
4461 : else
4462 : {
4463 : /* When there is no explicit mask on the call we have
4464 : more relaxed requirements. */
4465 113 : tree masktype;
4466 113 : poly_uint64 callee_nelements;
4467 113 : if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
4468 : {
4469 12 : callee_nelements
4470 12 : = exact_div (bestn_inbranch->simdclone->simdlen,
4471 : bestn_inbranch->simdclone->args[i].linear_step);
4472 12 : masktype = get_related_vectype_for_scalar_type
4473 12 : (vinfo->vector_mode, TREE_TYPE (vectype),
4474 : callee_nelements);
4475 : }
4476 : else
4477 : {
4478 101 : masktype = bestn_inbranch->simdclone->args[i].vector_type;
4479 : /* The aarch64 port will add custom attributes to types
4480 : for SVE simdclones which make the types different. We
4481 : should use canonincal types for masks within the
4482 : vectorizer, hence we construct the related vectype
4483 : here. */
4484 101 : masktype
4485 : = build_truth_vector_type_for_mode
4486 101 : (TYPE_VECTOR_SUBPARTS (masktype),
4487 101 : TYPE_MODE (masktype));
4488 101 : callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
4489 : }
4490 113 : auto o = vector_unroll_factor (nunits, callee_nelements);
4491 113 : vect_record_loop_mask (loop_vinfo,
4492 : &LOOP_VINFO_MASKS (loop_vinfo),
4493 : ncopies * o, masktype, NULL_TREE);
4494 : }
4495 : }
4496 1010 : else if ((bestn->simdclone->args[i].arg_type
4497 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4498 903 : || (bestn->simdclone->args[i].arg_type
4499 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
4500 892 : || (bestn_inbranch
4501 352 : && ((bestn_inbranch->simdclone->args[i].arg_type
4502 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4503 352 : || (bestn_inbranch->simdclone->args[i].arg_type
4504 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
4505 : {
4506 118 : simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
4507 118 : simd_clone_info.safe_push (arginfo[i].op);
4508 202 : tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
4509 202 : ? size_type_node : TREE_TYPE (arginfo[i].op));
4510 118 : tree ls = build_int_cst (lst, arginfo[i].linear_step);
4511 118 : simd_clone_info.safe_push (ls);
4512 118 : tree sll = (arginfo[i].simd_lane_linear
4513 118 : ? boolean_true_node : boolean_false_node);
4514 118 : simd_clone_info.safe_push (sll);
4515 : }
4516 : }
4517 :
4518 455 : SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
4519 455 : slp_node->data = new vect_simd_clone_data (std::move (_data));
4520 455 : DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4521 : /* ??? We're confused by calls w/o LHS. */
4522 455 : if (SLP_TREE_VECTYPE (slp_node))
4523 449 : vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
4524 455 : return true;
4525 : }
4526 :
4527 : /* Transform. */
4528 :
4529 358 : if (dump_enabled_p ())
4530 242 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
4531 :
4532 : /* Handle def. */
4533 358 : scalar_dest = gimple_call_lhs (stmt);
4534 358 : vec_dest = NULL_TREE;
4535 358 : rtype = NULL_TREE;
4536 358 : ratype = NULL_TREE;
4537 358 : if (scalar_dest)
4538 : {
4539 352 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
4540 352 : rtype = TREE_TYPE (TREE_TYPE (fndecl));
4541 352 : if (TREE_CODE (rtype) == ARRAY_TYPE)
4542 : {
4543 9 : ratype = rtype;
4544 9 : rtype = TREE_TYPE (ratype);
4545 : }
4546 : }
4547 :
4548 716 : auto_vec<vec<tree> > vec_oprnds;
4549 358 : auto_vec<unsigned> vec_oprnds_i;
4550 358 : vec_oprnds_i.safe_grow_cleared (nargs, true);
4551 358 : vec_oprnds.reserve_exact (nargs);
4552 358 : vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
4553 823 : for (j = 0; j < ncopies; ++j)
4554 : {
4555 465 : poly_uint64 callee_nelements;
4556 465 : poly_uint64 caller_nelements;
4557 : /* Build argument list for the vectorized call. */
4558 465 : if (j == 0)
4559 358 : vargs.create (nargs);
4560 : else
4561 107 : vargs.truncate (0);
4562 :
4563 1568 : for (i = 0; i < nargs; i++)
4564 : {
4565 1103 : unsigned int k, l, m, o;
4566 1103 : tree atype;
4567 1103 : tree op = gimple_call_arg (stmt, i + masked_call_offset);
4568 1103 : switch (bestn->simdclone->args[i].arg_type)
4569 : {
4570 814 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4571 814 : atype = bestn->simdclone->args[i].vector_type;
4572 814 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4573 814 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4574 814 : o = vector_unroll_factor (nunits, callee_nelements);
4575 1858 : for (m = j * o; m < (j + 1) * o; m++)
4576 : {
4577 1044 : if (known_lt (callee_nelements, caller_nelements))
4578 : {
4579 516 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
4580 258 : if (!constant_multiple_p (caller_nelements,
4581 : callee_nelements, &k))
4582 0 : gcc_unreachable ();
4583 :
4584 258 : gcc_assert ((k & (k - 1)) == 0);
4585 258 : if (m == 0)
4586 : {
4587 57 : vec_oprnds_i[i] = 0;
4588 57 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4589 : }
4590 : else
4591 : {
4592 201 : vec_oprnd0 = arginfo[i].op;
4593 201 : if ((m & (k - 1)) == 0)
4594 72 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4595 : }
4596 258 : arginfo[i].op = vec_oprnd0;
4597 258 : vec_oprnd0
4598 258 : = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
4599 258 : bitsize_int (prec),
4600 258 : bitsize_int ((m & (k - 1)) * prec));
4601 258 : gassign *new_stmt
4602 258 : = gimple_build_assign (make_ssa_name (atype),
4603 : vec_oprnd0);
4604 258 : vect_finish_stmt_generation (vinfo, stmt_info,
4605 : new_stmt, gsi);
4606 258 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4607 : }
4608 : else
4609 : {
4610 786 : if (!constant_multiple_p (callee_nelements,
4611 : caller_nelements, &k))
4612 0 : gcc_unreachable ();
4613 786 : gcc_assert ((k & (k - 1)) == 0);
4614 786 : vec<constructor_elt, va_gc> *ctor_elts;
4615 786 : if (k != 1)
4616 12 : vec_alloc (ctor_elts, k);
4617 : else
4618 774 : ctor_elts = NULL;
4619 810 : for (l = 0; l < k; l++)
4620 : {
4621 798 : if (m == 0 && l == 0)
4622 : {
4623 450 : vec_oprnds_i[i] = 0;
4624 450 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4625 : }
4626 : else
4627 348 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4628 798 : arginfo[i].op = vec_oprnd0;
4629 798 : if (k == 1)
4630 : break;
4631 24 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
4632 : vec_oprnd0);
4633 : }
4634 786 : if (k == 1)
4635 774 : if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
4636 : atype))
4637 : {
4638 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
4639 : vec_oprnd0);
4640 0 : gassign *new_stmt
4641 0 : = gimple_build_assign (make_ssa_name (atype),
4642 : vec_oprnd0);
4643 0 : vect_finish_stmt_generation (vinfo, stmt_info,
4644 : new_stmt, gsi);
4645 0 : vargs.safe_push (gimple_get_lhs (new_stmt));
4646 : }
4647 : else
4648 774 : vargs.safe_push (vec_oprnd0);
4649 : else
4650 : {
4651 12 : vec_oprnd0 = build_constructor (atype, ctor_elts);
4652 12 : gassign *new_stmt
4653 12 : = gimple_build_assign (make_ssa_name (atype),
4654 : vec_oprnd0);
4655 12 : vect_finish_stmt_generation (vinfo, stmt_info,
4656 : new_stmt, gsi);
4657 12 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4658 : }
4659 : }
4660 : }
4661 : break;
4662 66 : case SIMD_CLONE_ARG_TYPE_MASK:
4663 66 : if (bestn->simdclone->mask_mode == VOIDmode)
4664 : {
4665 60 : atype = bestn->simdclone->args[i].vector_type;
4666 60 : tree elt_type = TREE_TYPE (atype);
4667 60 : tree one = fold_convert (elt_type, integer_one_node);
4668 60 : tree zero = fold_convert (elt_type, integer_zero_node);
4669 60 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4670 60 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4671 60 : o = vector_unroll_factor (nunits, callee_nelements);
4672 120 : for (m = j * o; m < (j + 1) * o; m++)
4673 : {
4674 60 : if (maybe_lt (callee_nelements, caller_nelements))
4675 : {
4676 : /* The mask type has fewer elements than simdlen. */
4677 :
4678 : /* FORNOW */
4679 0 : gcc_unreachable ();
4680 : }
4681 60 : else if (known_eq (callee_nelements, caller_nelements))
4682 : {
4683 : /* The SIMD clone function has the same number of
4684 : elements as the current function. */
4685 60 : if (m == 0)
4686 60 : vec_oprnds_i[i] = 0;
4687 60 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4688 60 : if (loop_vinfo
4689 60 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4690 : {
4691 0 : vec_loop_masks *loop_masks
4692 : = &LOOP_VINFO_MASKS (loop_vinfo);
4693 0 : tree loop_mask
4694 0 : = vect_get_loop_mask (loop_vinfo, gsi,
4695 : loop_masks, ncopies_in,
4696 0 : vectype, j);
4697 0 : vec_oprnd0
4698 0 : = prepare_vec_mask (loop_vinfo,
4699 0 : TREE_TYPE (loop_mask),
4700 : loop_mask, vec_oprnd0,
4701 : gsi);
4702 0 : loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
4703 : loop_mask });
4704 :
4705 : }
4706 60 : vec_oprnd0
4707 60 : = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
4708 : build_vector_from_val (atype, one),
4709 : build_vector_from_val (atype, zero));
4710 60 : gassign *new_stmt
4711 60 : = gimple_build_assign (make_ssa_name (atype),
4712 : vec_oprnd0);
4713 60 : vect_finish_stmt_generation (vinfo, stmt_info,
4714 : new_stmt, gsi);
4715 60 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4716 : }
4717 : else
4718 : {
4719 : /* The mask type has more elements than simdlen. */
4720 :
4721 : /* FORNOW */
4722 0 : gcc_unreachable ();
4723 : }
4724 : }
4725 : }
4726 6 : else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4727 : {
4728 6 : atype = bestn->simdclone->args[i].vector_type;
4729 6 : poly_uint64 atype_subparts
4730 6 : = exact_div (bestn->simdclone->simdlen,
4731 : bestn->simdclone->args[i].linear_step);
4732 6 : o = bestn->simdclone->args[i].linear_step;
4733 12 : for (m = j * o; m < (j + 1) * o; m++)
4734 : {
4735 6 : if (m == 0)
4736 6 : vec_oprnds_i[i] = 0;
4737 6 : if (maybe_lt (atype_subparts,
4738 6 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4739 : {
4740 : /* The mask argument has fewer elements than the
4741 : input vector. */
4742 : /* FORNOW */
4743 0 : gcc_unreachable ();
4744 : }
4745 6 : else if (known_eq (atype_subparts,
4746 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4747 : {
4748 6 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4749 6 : if (loop_vinfo
4750 6 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4751 : {
4752 1 : vec_loop_masks *loop_masks
4753 : = &LOOP_VINFO_MASKS (loop_vinfo);
4754 1 : tree loop_mask
4755 1 : = vect_get_loop_mask (loop_vinfo, gsi,
4756 : loop_masks, ncopies_in,
4757 : vectype, j);
4758 1 : vec_oprnd0
4759 1 : = prepare_vec_mask (loop_vinfo,
4760 1 : TREE_TYPE (loop_mask),
4761 : loop_mask, vec_oprnd0,
4762 : gsi);
4763 : }
4764 : /* The vector mask argument matches the input
4765 : in the number of lanes, but not necessarily
4766 : in the mode. */
4767 6 : tree st = lang_hooks.types.type_for_mode
4768 6 : (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
4769 6 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
4770 : vec_oprnd0);
4771 6 : gassign *new_stmt
4772 6 : = gimple_build_assign (make_ssa_name (st),
4773 : vec_oprnd0);
4774 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4775 : new_stmt, gsi);
4776 6 : if (!types_compatible_p (atype, st))
4777 : {
4778 6 : new_stmt
4779 6 : = gimple_build_assign (make_ssa_name (atype),
4780 : NOP_EXPR,
4781 : gimple_assign_lhs
4782 : (new_stmt));
4783 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4784 : new_stmt, gsi);
4785 : }
4786 6 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4787 : }
4788 : else
4789 : {
4790 : /* The mask argument has more elements than the
4791 : input vector. */
4792 : /* FORNOW */
4793 0 : gcc_unreachable ();
4794 : }
4795 : }
4796 : }
4797 : else
4798 0 : gcc_unreachable ();
4799 : break;
4800 102 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4801 102 : vargs.safe_push (op);
4802 102 : break;
4803 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4804 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4805 121 : if (j == 0)
4806 : {
4807 118 : gimple_seq stmts;
4808 118 : arginfo[i].op
4809 118 : = force_gimple_operand (unshare_expr (arginfo[i].op),
4810 : &stmts, true, NULL_TREE);
4811 118 : if (stmts != NULL)
4812 : {
4813 0 : basic_block new_bb;
4814 0 : edge pe = loop_preheader_edge (loop);
4815 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4816 0 : gcc_assert (!new_bb);
4817 : }
4818 118 : if (arginfo[i].simd_lane_linear)
4819 : {
4820 6 : vargs.safe_push (arginfo[i].op);
4821 6 : break;
4822 : }
4823 112 : tree phi_res = copy_ssa_name (op);
4824 112 : gphi *new_phi = create_phi_node (phi_res, loop->header);
4825 112 : add_phi_arg (new_phi, arginfo[i].op,
4826 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
4827 112 : enum tree_code code
4828 196 : = POINTER_TYPE_P (TREE_TYPE (op))
4829 112 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4830 196 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4831 196 : ? sizetype : TREE_TYPE (op);
4832 112 : poly_widest_int cst
4833 112 : = wi::mul (bestn->simdclone->args[i].linear_step,
4834 112 : ncopies * nunits);
4835 112 : tree tcst = wide_int_to_tree (type, cst);
4836 112 : tree phi_arg = copy_ssa_name (op);
4837 112 : gassign *new_stmt
4838 112 : = gimple_build_assign (phi_arg, code, phi_res, tcst);
4839 112 : gimple_stmt_iterator si = gsi_after_labels (loop->header);
4840 112 : gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4841 112 : add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4842 : UNKNOWN_LOCATION);
4843 112 : arginfo[i].op = phi_res;
4844 112 : vargs.safe_push (phi_res);
4845 112 : }
4846 : else
4847 : {
4848 3 : enum tree_code code
4849 6 : = POINTER_TYPE_P (TREE_TYPE (op))
4850 3 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4851 6 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4852 6 : ? sizetype : TREE_TYPE (op);
4853 3 : poly_widest_int cst
4854 3 : = wi::mul (bestn->simdclone->args[i].linear_step,
4855 3 : j * nunits);
4856 3 : tree tcst = wide_int_to_tree (type, cst);
4857 3 : new_temp = make_ssa_name (TREE_TYPE (op));
4858 3 : gassign *new_stmt
4859 6 : = gimple_build_assign (new_temp, code,
4860 3 : arginfo[i].op, tcst);
4861 3 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4862 3 : vargs.safe_push (new_temp);
4863 3 : }
4864 : break;
4865 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4866 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4867 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4868 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4869 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4870 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4871 0 : default:
4872 0 : gcc_unreachable ();
4873 : }
4874 : }
4875 :
4876 465 : if (masked_call_offset == 0
4877 399 : && bestn->simdclone->inbranch
4878 13 : && bestn->simdclone->nargs > nargs)
4879 : {
4880 13 : unsigned long m, o;
4881 13 : size_t mask_i = bestn->simdclone->nargs - 1;
4882 13 : tree mask;
4883 13 : gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
4884 : SIMD_CLONE_ARG_TYPE_MASK);
4885 :
4886 13 : tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
4887 13 : tree mask_vectype;
4888 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4889 : {
4890 2 : callee_nelements = exact_div (bestn->simdclone->simdlen,
4891 : bestn->simdclone->args[i].linear_step);
4892 2 : mask_vectype = get_related_vectype_for_scalar_type
4893 2 : (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
4894 : }
4895 : else
4896 : {
4897 11 : mask_vectype = mask_argtype;
4898 11 : callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
4899 : }
4900 13 : o = vector_unroll_factor (nunits, callee_nelements);
4901 26 : for (m = j * o; m < (j + 1) * o; m++)
4902 : {
4903 13 : if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4904 : {
4905 1 : vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
4906 1 : mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
4907 : ncopies * o, mask_vectype, m);
4908 : }
4909 : else
4910 12 : mask = vect_build_all_ones_mask (vinfo, stmt_info,
4911 : mask_argtype);
4912 :
4913 13 : gassign *new_stmt;
4914 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4915 : {
4916 : /* This means we are dealing with integer mask modes.
4917 : First convert to an integer type with the same size as
4918 : the current vector type. */
4919 2 : unsigned HOST_WIDE_INT intermediate_size
4920 2 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
4921 2 : tree mid_int_type =
4922 2 : build_nonstandard_integer_type (intermediate_size, 1);
4923 2 : mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
4924 2 : new_stmt
4925 2 : = gimple_build_assign (make_ssa_name (mid_int_type),
4926 : mask);
4927 2 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4928 : /* Then zero-extend to the mask mode. */
4929 2 : mask = fold_build1 (NOP_EXPR, mask_argtype,
4930 : gimple_get_lhs (new_stmt));
4931 : }
4932 11 : else if (bestn->simdclone->mask_mode == VOIDmode)
4933 11 : mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
4934 : build_one_cst (mask_argtype),
4935 : build_zero_cst (mask_argtype));
4936 : else
4937 0 : gcc_unreachable ();
4938 :
4939 13 : new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
4940 : mask);
4941 13 : vect_finish_stmt_generation (vinfo, stmt_info,
4942 : new_stmt, gsi);
4943 13 : mask = gimple_assign_lhs (new_stmt);
4944 13 : vargs.safe_push (mask);
4945 : }
4946 : }
4947 :
4948 465 : gcall *new_call = gimple_build_call_vec (fndecl, vargs);
4949 465 : if (vec_dest)
4950 : {
4951 459 : gcc_assert (ratype
4952 : || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
4953 459 : if (ratype)
4954 15 : new_temp = create_tmp_var (ratype);
4955 444 : else if (useless_type_conversion_p (vectype, rtype))
4956 424 : new_temp = make_ssa_name (vec_dest, new_call);
4957 : else
4958 20 : new_temp = make_ssa_name (rtype, new_call);
4959 459 : gimple_call_set_lhs (new_call, new_temp);
4960 : }
4961 465 : vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
4962 465 : gimple *new_stmt = new_call;
4963 :
4964 465 : if (vec_dest)
4965 : {
4966 459 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
4967 : {
4968 19 : unsigned int k, l;
4969 38 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
4970 38 : poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
4971 19 : k = vector_unroll_factor (nunits,
4972 : TYPE_VECTOR_SUBPARTS (vectype));
4973 19 : gcc_assert ((k & (k - 1)) == 0);
4974 69 : for (l = 0; l < k; l++)
4975 : {
4976 50 : tree t;
4977 50 : if (ratype)
4978 : {
4979 42 : t = build_fold_addr_expr (new_temp);
4980 42 : t = build2 (MEM_REF, vectype, t,
4981 42 : build_int_cst (TREE_TYPE (t), l * bytes));
4982 : }
4983 : else
4984 8 : t = build3 (BIT_FIELD_REF, vectype, new_temp,
4985 8 : bitsize_int (prec), bitsize_int (l * prec));
4986 50 : new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
4987 50 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4988 :
4989 50 : SLP_TREE_VEC_DEFS (slp_node)
4990 50 : .quick_push (gimple_assign_lhs (new_stmt));
4991 : }
4992 :
4993 19 : if (ratype)
4994 15 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
4995 19 : continue;
4996 19 : }
4997 440 : else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
4998 : {
4999 16 : unsigned int k;
5000 16 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
5001 16 : TYPE_VECTOR_SUBPARTS (rtype), &k))
5002 0 : gcc_unreachable ();
5003 16 : gcc_assert ((k & (k - 1)) == 0);
5004 16 : if ((j & (k - 1)) == 0)
5005 8 : vec_alloc (ret_ctor_elts, k);
5006 16 : if (ratype)
5007 : {
5008 0 : unsigned int m, o;
5009 0 : o = vector_unroll_factor (nunits,
5010 : TYPE_VECTOR_SUBPARTS (rtype));
5011 0 : for (m = 0; m < o; m++)
5012 : {
5013 0 : tree tem = build4 (ARRAY_REF, rtype, new_temp,
5014 0 : size_int (m), NULL_TREE, NULL_TREE);
5015 0 : new_stmt = gimple_build_assign (make_ssa_name (rtype),
5016 : tem);
5017 0 : vect_finish_stmt_generation (vinfo, stmt_info,
5018 : new_stmt, gsi);
5019 0 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
5020 : gimple_assign_lhs (new_stmt));
5021 : }
5022 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5023 : }
5024 : else
5025 16 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
5026 16 : if ((j & (k - 1)) != k - 1)
5027 8 : continue;
5028 8 : vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
5029 8 : new_stmt
5030 8 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5031 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5032 :
5033 8 : SLP_TREE_VEC_DEFS (slp_node)
5034 8 : .quick_push (gimple_assign_lhs (new_stmt));
5035 8 : continue;
5036 8 : }
5037 424 : else if (ratype)
5038 : {
5039 0 : tree t = build_fold_addr_expr (new_temp);
5040 0 : t = build2 (MEM_REF, vectype, t,
5041 0 : build_int_cst (TREE_TYPE (t), 0));
5042 0 : new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
5043 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5044 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5045 : }
5046 424 : else if (!useless_type_conversion_p (vectype, rtype))
5047 : {
5048 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
5049 0 : new_stmt
5050 0 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5051 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5052 : }
5053 : }
5054 :
5055 430 : if (gimple_get_lhs (new_stmt))
5056 424 : SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
5057 : }
5058 :
5059 1151 : for (i = 0; i < nargs; ++i)
5060 : {
5061 793 : vec<tree> oprndsi = vec_oprnds[i];
5062 793 : oprndsi.release ();
5063 : }
5064 358 : vargs.release ();
5065 :
5066 : /* Mark the clone as no longer being a candidate for GC. */
5067 358 : bestn->gc_candidate = false;
5068 :
5069 358 : return true;
5070 1396 : }
5071 :
5072 :
5073 : /* Function vect_gen_widened_results_half
5074 :
5075 : Create a vector stmt whose code, type, number of arguments, and result
5076 : variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5077 : VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5078 : In the case that CODE is a CALL_EXPR, this means that a call to DECL
5079 : needs to be created (DECL is a function-decl of a target-builtin).
5080 : STMT_INFO is the original scalar stmt that we are vectorizing. */
5081 :
5082 : static gimple *
5083 32206 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
5084 : tree vec_oprnd0, tree vec_oprnd1, int op_type,
5085 : tree vec_dest, gimple_stmt_iterator *gsi,
5086 : stmt_vec_info stmt_info)
5087 : {
5088 32206 : gimple *new_stmt;
5089 32206 : tree new_temp;
5090 :
5091 : /* Generate half of the widened result: */
5092 32206 : if (op_type != binary_op)
5093 31096 : vec_oprnd1 = NULL;
5094 32206 : new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
5095 32206 : new_temp = make_ssa_name (vec_dest, new_stmt);
5096 32206 : gimple_set_lhs (new_stmt, new_temp);
5097 32206 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5098 :
5099 32206 : return new_stmt;
5100 : }
5101 :
5102 :
5103 : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5104 : For multi-step conversions store the resulting vectors and call the function
5105 : recursively. When NARROW_SRC_P is true, there's still a conversion after
5106 : narrowing, don't store the vectors in the SLP_NODE or in vector info of
5107 : the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5108 :
5109 : static void
5110 12086 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
5111 : int multi_step_cvt,
5112 : stmt_vec_info stmt_info,
5113 : vec<tree> &vec_dsts,
5114 : gimple_stmt_iterator *gsi,
5115 : slp_tree slp_node, code_helper code,
5116 : bool narrow_src_p)
5117 : {
5118 12086 : unsigned int i;
5119 12086 : tree vop0, vop1, new_tmp, vec_dest;
5120 :
5121 12086 : vec_dest = vec_dsts.pop ();
5122 :
5123 28592 : for (i = 0; i < vec_oprnds->length (); i += 2)
5124 : {
5125 : /* Create demotion operation. */
5126 16506 : vop0 = (*vec_oprnds)[i];
5127 16506 : vop1 = (*vec_oprnds)[i + 1];
5128 16506 : gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
5129 16506 : new_tmp = make_ssa_name (vec_dest, new_stmt);
5130 16506 : gimple_set_lhs (new_stmt, new_tmp);
5131 16506 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5132 16506 : if (multi_step_cvt || narrow_src_p)
5133 : /* Store the resulting vector for next recursive call,
5134 : or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5135 6770 : (*vec_oprnds)[i/2] = new_tmp;
5136 : else
5137 : {
5138 : /* This is the last step of the conversion sequence. Store the
5139 : vectors in SLP_NODE. */
5140 9736 : slp_node->push_vec_def (new_stmt);
5141 : }
5142 : }
5143 :
5144 : /* For multi-step demotion operations we first generate demotion operations
5145 : from the source type to the intermediate types, and then combine the
5146 : results (stored in VEC_OPRNDS) in demotion operation to the destination
5147 : type. */
5148 12086 : if (multi_step_cvt)
5149 : {
5150 : /* At each level of recursion we have half of the operands we had at the
5151 : previous level. */
5152 3011 : vec_oprnds->truncate ((i+1)/2);
5153 3011 : vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
5154 : multi_step_cvt - 1,
5155 : stmt_info, vec_dsts, gsi,
5156 3011 : slp_node, VEC_PACK_TRUNC_EXPR,
5157 : narrow_src_p);
5158 : }
5159 :
5160 12086 : vec_dsts.quick_push (vec_dest);
5161 12086 : }
5162 :
5163 :
5164 : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5165 : and VEC_OPRNDS1, for a binary operation associated with scalar statement
5166 : STMT_INFO. For multi-step conversions store the resulting vectors and
5167 : call the function recursively. */
5168 :
5169 : static void
5170 11724 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
5171 : vec<tree> *vec_oprnds0,
5172 : vec<tree> *vec_oprnds1,
5173 : stmt_vec_info stmt_info, tree vec_dest,
5174 : gimple_stmt_iterator *gsi,
5175 : code_helper ch1,
5176 : code_helper ch2, int op_type)
5177 : {
5178 11724 : int i;
5179 11724 : tree vop0, vop1, new_tmp1, new_tmp2;
5180 11724 : gimple *new_stmt1, *new_stmt2;
5181 11724 : vec<tree> vec_tmp = vNULL;
5182 :
5183 11724 : vec_tmp.create (vec_oprnds0->length () * 2);
5184 39551 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5185 : {
5186 16103 : if (op_type == binary_op)
5187 555 : vop1 = (*vec_oprnds1)[i];
5188 : else
5189 : vop1 = NULL_TREE;
5190 :
5191 : /* Generate the two halves of promotion operation. */
5192 16103 : new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
5193 : op_type, vec_dest, gsi,
5194 : stmt_info);
5195 16103 : new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
5196 : op_type, vec_dest, gsi,
5197 : stmt_info);
5198 16103 : if (is_gimple_call (new_stmt1))
5199 : {
5200 0 : new_tmp1 = gimple_call_lhs (new_stmt1);
5201 0 : new_tmp2 = gimple_call_lhs (new_stmt2);
5202 : }
5203 : else
5204 : {
5205 16103 : new_tmp1 = gimple_assign_lhs (new_stmt1);
5206 16103 : new_tmp2 = gimple_assign_lhs (new_stmt2);
5207 : }
5208 :
5209 : /* Store the results for the next step. */
5210 16103 : vec_tmp.quick_push (new_tmp1);
5211 16103 : vec_tmp.quick_push (new_tmp2);
5212 : }
5213 :
5214 11724 : vec_oprnds0->release ();
5215 11724 : *vec_oprnds0 = vec_tmp;
5216 11724 : }
5217 :
5218 : /* Create vectorized promotion stmts for widening stmts using only half the
5219 : potential vector size for input. */
5220 : static void
5221 14 : vect_create_half_widening_stmts (vec_info *vinfo,
5222 : vec<tree> *vec_oprnds0,
5223 : vec<tree> *vec_oprnds1,
5224 : stmt_vec_info stmt_info, tree vec_dest,
5225 : gimple_stmt_iterator *gsi,
5226 : code_helper code1,
5227 : int op_type)
5228 : {
5229 14 : int i;
5230 14 : tree vop0, vop1;
5231 14 : gimple *new_stmt1;
5232 14 : gimple *new_stmt2;
5233 14 : gimple *new_stmt3;
5234 14 : vec<tree> vec_tmp = vNULL;
5235 :
5236 14 : vec_tmp.create (vec_oprnds0->length ());
5237 28 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5238 : {
5239 14 : tree new_tmp1, new_tmp2, new_tmp3, out_type;
5240 :
5241 14 : gcc_assert (op_type == binary_op);
5242 14 : vop1 = (*vec_oprnds1)[i];
5243 :
5244 : /* Widen the first vector input. */
5245 14 : out_type = TREE_TYPE (vec_dest);
5246 14 : new_tmp1 = make_ssa_name (out_type);
5247 14 : new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
5248 14 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
5249 14 : if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
5250 : {
5251 : /* Widen the second vector input. */
5252 14 : new_tmp2 = make_ssa_name (out_type);
5253 14 : new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
5254 14 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
5255 : /* Perform the operation. With both vector inputs widened. */
5256 14 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
5257 : }
5258 : else
5259 : {
5260 : /* Perform the operation. With the single vector input widened. */
5261 0 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
5262 : }
5263 :
5264 14 : new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
5265 14 : gimple_assign_set_lhs (new_stmt3, new_tmp3);
5266 14 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
5267 :
5268 : /* Store the results for the next step. */
5269 14 : vec_tmp.quick_push (new_tmp3);
5270 : }
5271 :
5272 14 : vec_oprnds0->release ();
5273 14 : *vec_oprnds0 = vec_tmp;
5274 14 : }
5275 :
5276 :
5277 : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5278 : If COST_VEC is passed, calculate costs but don't change anything,
5279 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5280 : it, and insert it at GSI.
5281 : Return true if STMT_INFO is vectorizable in this way. */
5282 :
5283 : static bool
5284 2730227 : vectorizable_conversion (vec_info *vinfo,
5285 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5286 : slp_tree slp_node,
5287 : stmt_vector_for_cost *cost_vec)
5288 : {
5289 2730227 : tree vec_dest, cvt_op = NULL_TREE;
5290 2730227 : tree scalar_dest;
5291 2730227 : tree op0, op1 = NULL_TREE;
5292 2730227 : tree_code tc1;
5293 2730227 : code_helper code, code1, code2;
5294 2730227 : code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
5295 2730227 : tree new_temp;
5296 2730227 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5297 2730227 : poly_uint64 nunits_in;
5298 2730227 : poly_uint64 nunits_out;
5299 2730227 : tree vectype_out, vectype_in;
5300 2730227 : int i;
5301 2730227 : tree lhs_type, rhs_type;
5302 : /* For conversions between floating point and integer, there're 2 NARROW
5303 : cases. NARROW_SRC is for FLOAT_EXPR, means
5304 : integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5305 : This is safe when the range of the source integer can fit into the lower
5306 : precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5307 : floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5308 : For other conversions, when there's narrowing, NARROW_DST is used as
5309 : default. */
5310 2730227 : enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
5311 2730227 : vec<tree> vec_oprnds0 = vNULL;
5312 2730227 : vec<tree> vec_oprnds1 = vNULL;
5313 2730227 : tree vop0;
5314 2730227 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5315 2730227 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5316 2730227 : int multi_step_cvt = 0;
5317 2730227 : vec<tree> interm_types = vNULL;
5318 2730227 : tree intermediate_type, cvt_type = NULL_TREE;
5319 2730227 : int op_type;
5320 2730227 : unsigned short fltsz;
5321 :
5322 : /* Is STMT a vectorizable conversion? */
5323 :
5324 2730227 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5325 : return false;
5326 :
5327 2730227 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5328 237977 : && cost_vec)
5329 : return false;
5330 :
5331 2492250 : gimple* stmt = stmt_info->stmt;
5332 2492250 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
5333 : return false;
5334 :
5335 2432896 : if (gimple_get_lhs (stmt) == NULL_TREE
5336 2432896 : || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5337 826522 : return false;
5338 :
5339 1606374 : if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5340 : return false;
5341 :
5342 1606374 : if (is_gimple_assign (stmt))
5343 : {
5344 1593594 : code = gimple_assign_rhs_code (stmt);
5345 1593594 : op_type = TREE_CODE_LENGTH ((tree_code) code);
5346 : }
5347 12780 : else if (gimple_call_internal_p (stmt))
5348 : {
5349 7876 : code = gimple_call_internal_fn (stmt);
5350 7876 : op_type = gimple_call_num_args (stmt);
5351 : }
5352 : else
5353 : return false;
5354 :
5355 1601470 : bool widen_arith = (code == WIDEN_MULT_EXPR
5356 1599117 : || code == WIDEN_LSHIFT_EXPR
5357 3200587 : || widening_fn_p (code));
5358 :
5359 1599117 : if (!widen_arith
5360 1599117 : && !CONVERT_EXPR_CODE_P (code)
5361 1434336 : && code != FIX_TRUNC_EXPR
5362 1432590 : && code != FLOAT_EXPR)
5363 : return false;
5364 :
5365 : /* Check types of lhs and rhs. */
5366 187267 : scalar_dest = gimple_get_lhs (stmt);
5367 187267 : lhs_type = TREE_TYPE (scalar_dest);
5368 187267 : vectype_out = SLP_TREE_VECTYPE (slp_node);
5369 :
5370 : /* Check the operands of the operation. */
5371 187267 : slp_tree slp_op0, slp_op1 = NULL;
5372 187267 : if (!vect_is_simple_use (vinfo, slp_node,
5373 : 0, &op0, &slp_op0, &dt[0], &vectype_in))
5374 : {
5375 0 : if (dump_enabled_p ())
5376 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5377 : "use not simple.\n");
5378 0 : return false;
5379 : }
5380 :
5381 187267 : rhs_type = TREE_TYPE (op0);
5382 185521 : if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
5383 358188 : && !((INTEGRAL_TYPE_P (lhs_type)
5384 157105 : && INTEGRAL_TYPE_P (rhs_type))
5385 : || (SCALAR_FLOAT_TYPE_P (lhs_type)
5386 8905 : && SCALAR_FLOAT_TYPE_P (rhs_type))))
5387 : return false;
5388 :
5389 182356 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5390 162264 : && INTEGRAL_TYPE_P (lhs_type)
5391 317328 : && !type_has_mode_precision_p (lhs_type))
5392 : {
5393 495 : if (dump_enabled_p ())
5394 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5395 : "type conversion to bit-precision unsupported\n");
5396 495 : return false;
5397 : }
5398 :
5399 181861 : if (op_type == binary_op)
5400 : {
5401 2353 : gcc_assert (code == WIDEN_MULT_EXPR
5402 : || code == WIDEN_LSHIFT_EXPR
5403 : || widening_fn_p (code));
5404 :
5405 2353 : op1 = is_gimple_assign (stmt) ? gimple_assign_rhs2 (stmt) :
5406 0 : gimple_call_arg (stmt, 0);
5407 2353 : tree vectype1_in;
5408 2353 : if (!vect_is_simple_use (vinfo, slp_node, 1,
5409 : &op1, &slp_op1, &dt[1], &vectype1_in))
5410 : {
5411 0 : if (dump_enabled_p ())
5412 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5413 : "use not simple.\n");
5414 0 : return false;
5415 : }
5416 : /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5417 : OP1. */
5418 2353 : if (!vectype_in)
5419 106 : vectype_in = vectype1_in;
5420 : }
5421 :
5422 : /* If op0 is an external or constant def, infer the vector type
5423 : from the scalar type. */
5424 181861 : if (!vectype_in)
5425 20812 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
5426 181861 : if (!cost_vec)
5427 23004 : gcc_assert (vectype_in);
5428 181861 : if (!vectype_in)
5429 : {
5430 260 : if (dump_enabled_p ())
5431 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5432 : "no vectype for scalar type %T\n", rhs_type);
5433 :
5434 260 : return false;
5435 : }
5436 :
5437 363202 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
5438 181601 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
5439 : {
5440 261 : if (dump_enabled_p ())
5441 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5442 : "can't convert between boolean and non "
5443 : "boolean vectors %T\n", rhs_type);
5444 :
5445 261 : return false;
5446 : }
5447 :
5448 181340 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
5449 181340 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5450 181340 : if (known_eq (nunits_out, nunits_in))
5451 86708 : if (widen_arith)
5452 : modifier = WIDEN;
5453 : else
5454 181340 : modifier = NONE;
5455 94632 : else if (multiple_p (nunits_out, nunits_in))
5456 : modifier = NARROW_DST;
5457 : else
5458 : {
5459 52281 : gcc_checking_assert (multiple_p (nunits_in, nunits_out));
5460 : modifier = WIDEN;
5461 : }
5462 :
5463 181340 : bool found_mode = false;
5464 181340 : scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
5465 181340 : scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
5466 181340 : opt_scalar_mode rhs_mode_iter;
5467 181340 : auto_vec<std::pair<tree, tree_code>, 2> converts;
5468 181340 : bool evenodd_ok = false;
5469 :
5470 : /* Supportable by target? */
5471 181340 : switch (modifier)
5472 : {
5473 86458 : case NONE:
5474 86458 : if (code != FIX_TRUNC_EXPR
5475 85406 : && code != FLOAT_EXPR
5476 162789 : && !CONVERT_EXPR_CODE_P (code))
5477 : return false;
5478 86458 : gcc_assert (code.is_tree_code ());
5479 86458 : if (supportable_indirect_convert_operation (code,
5480 : vectype_out, vectype_in,
5481 : converts, op0, slp_op0))
5482 : {
5483 19541 : gcc_assert (converts.length () <= 2);
5484 19541 : if (converts.length () == 1)
5485 19467 : code1 = converts[0].second;
5486 : else
5487 : {
5488 74 : cvt_type = NULL_TREE;
5489 74 : multi_step_cvt = converts.length () - 1;
5490 74 : codecvt1 = converts[0].second;
5491 74 : code1 = converts[1].second;
5492 74 : interm_types.safe_push (converts[0].first);
5493 : }
5494 : break;
5495 : }
5496 :
5497 : /* FALLTHRU */
5498 66917 : unsupported:
5499 74149 : if (dump_enabled_p ())
5500 6078 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5501 : "conversion not supported by target.\n");
5502 : return false;
5503 :
5504 52531 : case WIDEN:
5505 52531 : if (known_eq (nunits_in, nunits_out))
5506 : {
5507 500 : if (!(code.is_tree_code ()
5508 250 : && supportable_half_widening_operation ((tree_code) code,
5509 : vectype_out, vectype_in,
5510 : &tc1)))
5511 73 : goto unsupported;
5512 177 : code1 = tc1;
5513 177 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5514 : break;
5515 : }
5516 : /* Elements in a vector can only be reordered if used in a reduction
5517 : operation only. */
5518 52281 : if (code == WIDEN_MULT_EXPR
5519 2103 : && loop_vinfo
5520 2056 : && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
5521 : /* For a SLP reduction we cannot swizzle lanes, detecting a
5522 : reduction chain isn't possible here. */
5523 54315 : && SLP_TREE_LANES (slp_node) == 1)
5524 : {
5525 : /* ??? There is no way to look for SLP uses, so work on
5526 : the stmt and what the stmt-based cycle detection gives us. */
5527 1932 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5528 1932 : stmt_vec_info use_stmt_info
5529 1932 : = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
5530 1932 : if (use_stmt_info
5531 1783 : && STMT_VINFO_REDUC_DEF (use_stmt_info))
5532 52281 : evenodd_ok = true;
5533 : }
5534 52281 : if (supportable_widening_operation (code, vectype_out, vectype_in,
5535 : evenodd_ok, &code1,
5536 : &code2, &multi_step_cvt,
5537 : &interm_types))
5538 : {
5539 : /* Binary widening operation can only be supported directly by the
5540 : architecture. */
5541 50335 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5542 : break;
5543 : }
5544 :
5545 1946 : if (code != FLOAT_EXPR
5546 2318 : || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
5547 1760 : goto unsupported;
5548 :
5549 186 : fltsz = GET_MODE_SIZE (lhs_mode);
5550 273 : FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
5551 : {
5552 273 : rhs_mode = rhs_mode_iter.require ();
5553 546 : if (GET_MODE_SIZE (rhs_mode) > fltsz)
5554 : break;
5555 :
5556 273 : cvt_type
5557 273 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5558 273 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5559 273 : if (cvt_type == NULL_TREE)
5560 0 : goto unsupported;
5561 :
5562 546 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5563 : {
5564 81 : tc1 = ERROR_MARK;
5565 81 : gcc_assert (code.is_tree_code ());
5566 81 : if (!supportable_convert_operation ((tree_code) code, vectype_out,
5567 : cvt_type, &tc1))
5568 22 : goto unsupported;
5569 59 : codecvt1 = tc1;
5570 : }
5571 192 : else if (!supportable_widening_operation (code, vectype_out,
5572 : cvt_type, evenodd_ok,
5573 : &codecvt1,
5574 : &codecvt2, &multi_step_cvt,
5575 : &interm_types))
5576 87 : continue;
5577 : else
5578 105 : gcc_assert (multi_step_cvt == 0);
5579 :
5580 164 : if (supportable_widening_operation (NOP_EXPR, cvt_type,
5581 : vectype_in, evenodd_ok, &code1,
5582 : &code2, &multi_step_cvt,
5583 : &interm_types))
5584 : {
5585 : found_mode = true;
5586 : break;
5587 : }
5588 : }
5589 :
5590 164 : if (!found_mode)
5591 0 : goto unsupported;
5592 :
5593 328 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5594 59 : codecvt2 = ERROR_MARK;
5595 : else
5596 : {
5597 105 : multi_step_cvt++;
5598 105 : interm_types.safe_push (cvt_type);
5599 105 : cvt_type = NULL_TREE;
5600 : }
5601 : break;
5602 :
5603 42351 : case NARROW_DST:
5604 42351 : gcc_assert (op_type == unary_op);
5605 42351 : if (supportable_narrowing_operation (code, vectype_out, vectype_in,
5606 : &code1, &multi_step_cvt,
5607 : &interm_types))
5608 : break;
5609 :
5610 16533 : if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
5611 984 : goto unsupported;
5612 :
5613 4527 : if (code == FIX_TRUNC_EXPR)
5614 : {
5615 107 : cvt_type
5616 107 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5617 107 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5618 107 : if (cvt_type == NULL_TREE)
5619 0 : goto unsupported;
5620 107 : if (supportable_convert_operation ((tree_code) code, cvt_type, vectype_in,
5621 : &tc1))
5622 105 : codecvt1 = tc1;
5623 : else
5624 2 : goto unsupported;
5625 105 : if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
5626 : &code1, &multi_step_cvt,
5627 : &interm_types))
5628 : break;
5629 : }
5630 : /* If op0 can be represented with low precision integer,
5631 : truncate it to cvt_type and the do FLOAT_EXPR. */
5632 4420 : else if (code == FLOAT_EXPR)
5633 : {
5634 143 : if (cost_vec)
5635 : {
5636 135 : wide_int op_min_value, op_max_value;
5637 135 : tree def;
5638 :
5639 : /* ??? Merge ranges in case of more than one lane. */
5640 135 : if (SLP_TREE_LANES (slp_op0) != 1
5641 133 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
5642 268 : || !vect_get_range_info (def, &op_min_value, &op_max_value))
5643 106 : goto unsupported;
5644 :
5645 29 : if ((wi::min_precision (op_max_value, SIGNED)
5646 29 : > GET_MODE_BITSIZE (lhs_mode))
5647 29 : || (wi::min_precision (op_min_value, SIGNED)
5648 27 : > GET_MODE_BITSIZE (lhs_mode)))
5649 2 : goto unsupported;
5650 135 : }
5651 :
5652 35 : cvt_type
5653 35 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
5654 35 : cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
5655 35 : if (cvt_type == NULL_TREE)
5656 0 : goto unsupported;
5657 35 : if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
5658 : &code1, &multi_step_cvt,
5659 : &interm_types))
5660 2 : goto unsupported;
5661 33 : if (supportable_convert_operation ((tree_code) code, vectype_out,
5662 : cvt_type, &tc1))
5663 : {
5664 33 : codecvt1 = tc1;
5665 33 : modifier = NARROW_SRC;
5666 33 : break;
5667 : }
5668 : }
5669 :
5670 4281 : goto unsupported;
5671 :
5672 : default:
5673 : gcc_unreachable ();
5674 : }
5675 :
5676 107191 : if (modifier == WIDEN
5677 107191 : && loop_vinfo
5678 49474 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
5679 128261 : && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
5680 21048 : || widening_evenodd_fn_p (code1)))
5681 : {
5682 22 : if (dump_enabled_p ())
5683 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5684 : "can't use a fully-masked loop because"
5685 : " widening operation on even/odd elements"
5686 : " mixes up lanes.\n");
5687 22 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
5688 : }
5689 :
5690 107191 : if (cost_vec) /* transformation not required. */
5691 : {
5692 84187 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
5693 84187 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
5694 : {
5695 0 : if (dump_enabled_p ())
5696 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5697 : "incompatible vector types for invariants\n");
5698 0 : return false;
5699 : }
5700 84187 : DUMP_VECT_SCOPE ("vectorizable_conversion");
5701 84187 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
5702 84187 : if (modifier == NONE)
5703 : {
5704 15517 : SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
5705 15517 : vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
5706 : slp_node, cost_vec);
5707 : }
5708 68670 : else if (modifier == NARROW_SRC || modifier == NARROW_DST)
5709 : {
5710 27899 : SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
5711 : /* The final packing step produces one vector result per copy. */
5712 27899 : vect_model_promotion_demotion_cost (slp_node, nvectors,
5713 : multi_step_cvt, cost_vec,
5714 : widen_arith);
5715 : }
5716 : else
5717 : {
5718 40771 : SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
5719 : /* The initial unpacking step produces two vector results
5720 : per copy. MULTI_STEP_CVT is 0 for a single conversion,
5721 : so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5722 40771 : vect_model_promotion_demotion_cost (slp_node,
5723 : nvectors >> multi_step_cvt,
5724 : multi_step_cvt, cost_vec,
5725 : widen_arith);
5726 : }
5727 84187 : interm_types.release ();
5728 84187 : return true;
5729 84187 : }
5730 :
5731 : /* Transform. */
5732 23004 : if (dump_enabled_p ())
5733 4279 : dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
5734 :
5735 23004 : if (op_type == binary_op)
5736 : {
5737 508 : if (CONSTANT_CLASS_P (op0))
5738 0 : op0 = fold_convert (TREE_TYPE (op1), op0);
5739 508 : else if (CONSTANT_CLASS_P (op1))
5740 234 : op1 = fold_convert (TREE_TYPE (op0), op1);
5741 : }
5742 :
5743 : /* In case of multi-step conversion, we first generate conversion operations
5744 : to the intermediate types, and then from that types to the final one.
5745 : We create vector destinations for the intermediate type (TYPES) received
5746 : from supportable_*_operation, and store them in the correct order
5747 : for future use in vect_create_vectorized_*_stmts (). */
5748 23004 : auto_vec<tree> vec_dsts (multi_step_cvt + 1);
5749 23004 : bool widen_or_narrow_float_p
5750 23004 : = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
5751 23004 : vec_dest = vect_create_destination_var (scalar_dest,
5752 : widen_or_narrow_float_p
5753 : ? cvt_type : vectype_out);
5754 23004 : vec_dsts.quick_push (vec_dest);
5755 :
5756 23004 : if (multi_step_cvt)
5757 : {
5758 9240 : for (i = interm_types.length () - 1;
5759 9240 : interm_types.iterate (i, &intermediate_type); i--)
5760 : {
5761 4865 : vec_dest = vect_create_destination_var (scalar_dest,
5762 : intermediate_type);
5763 4865 : vec_dsts.quick_push (vec_dest);
5764 : }
5765 : }
5766 :
5767 23004 : if (cvt_type)
5768 76 : vec_dest = vect_create_destination_var (scalar_dest,
5769 : widen_or_narrow_float_p
5770 : ? vectype_out : cvt_type);
5771 :
5772 23004 : switch (modifier)
5773 : {
5774 4024 : case NONE:
5775 4024 : vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
5776 : /* vec_dest is intermediate type operand when multi_step_cvt. */
5777 4024 : if (multi_step_cvt)
5778 : {
5779 21 : cvt_op = vec_dest;
5780 21 : vec_dest = vec_dsts[0];
5781 : }
5782 :
5783 8436 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5784 : {
5785 : /* Arguments are ready, create the new vector stmt. */
5786 4412 : gimple* new_stmt;
5787 4412 : if (multi_step_cvt)
5788 : {
5789 21 : gcc_assert (multi_step_cvt == 1);
5790 21 : new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
5791 21 : new_temp = make_ssa_name (cvt_op, new_stmt);
5792 21 : gimple_assign_set_lhs (new_stmt, new_temp);
5793 21 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5794 21 : vop0 = new_temp;
5795 : }
5796 4412 : new_stmt = vect_gimple_build (vec_dest, code1, vop0);
5797 4412 : new_temp = make_ssa_name (vec_dest, new_stmt);
5798 4412 : gimple_set_lhs (new_stmt, new_temp);
5799 4412 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5800 :
5801 4412 : slp_node->push_vec_def (new_stmt);
5802 : }
5803 : break;
5804 :
5805 9905 : case WIDEN:
5806 : /* In case the vectorization factor (VF) is bigger than the number
5807 : of elements that we can fit in a vectype (nunits), we have to
5808 : generate more than one vector stmt - i.e - we need to "unroll"
5809 : the vector stmt by a factor VF/nunits. */
5810 9905 : vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0,
5811 9905 : code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
5812 : &vec_oprnds1);
5813 9905 : if (code == WIDEN_LSHIFT_EXPR)
5814 : {
5815 0 : int oprnds_size = vec_oprnds0.length ();
5816 0 : vec_oprnds1.create (oprnds_size);
5817 0 : for (i = 0; i < oprnds_size; ++i)
5818 0 : vec_oprnds1.quick_push (op1);
5819 : }
5820 : /* Arguments are ready. Create the new vector stmts. */
5821 21643 : for (i = multi_step_cvt; i >= 0; i--)
5822 : {
5823 11738 : tree this_dest = vec_dsts[i];
5824 11738 : code_helper c1 = code1, c2 = code2;
5825 11738 : if (i == 0 && codecvt2 != ERROR_MARK)
5826 : {
5827 48 : c1 = codecvt1;
5828 48 : c2 = codecvt2;
5829 : }
5830 11738 : if (known_eq (nunits_out, nunits_in))
5831 14 : vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
5832 : stmt_info, this_dest, gsi, c1,
5833 : op_type);
5834 : else
5835 11724 : vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
5836 : &vec_oprnds1, stmt_info,
5837 : this_dest, gsi,
5838 : c1, c2, op_type);
5839 : }
5840 :
5841 37847 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5842 : {
5843 27942 : gimple *new_stmt;
5844 27942 : if (cvt_type)
5845 : {
5846 120 : new_temp = make_ssa_name (vec_dest);
5847 120 : new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5848 120 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5849 : }
5850 : else
5851 27822 : new_stmt = SSA_NAME_DEF_STMT (vop0);
5852 :
5853 27942 : slp_node->push_vec_def (new_stmt);
5854 : }
5855 : break;
5856 :
5857 9075 : case NARROW_SRC:
5858 9075 : case NARROW_DST:
5859 : /* In case the vectorization factor (VF) is bigger than the number
5860 : of elements that we can fit in a vectype (nunits), we have to
5861 : generate more than one vector stmt - i.e - we need to "unroll"
5862 : the vector stmt by a factor VF/nunits. */
5863 9075 : vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
5864 : /* Arguments are ready. Create the new vector stmts. */
5865 9075 : if (cvt_type && modifier == NARROW_DST)
5866 153 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5867 : {
5868 124 : new_temp = make_ssa_name (vec_dest);
5869 124 : gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5870 124 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5871 124 : vec_oprnds0[i] = new_temp;
5872 : }
5873 :
5874 9075 : vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
5875 : multi_step_cvt,
5876 : stmt_info, vec_dsts, gsi,
5877 : slp_node, code1,
5878 : modifier == NARROW_SRC);
5879 : /* After demoting op0 to cvt_type, convert it to dest. */
5880 9075 : if (cvt_type && code == FLOAT_EXPR)
5881 : {
5882 16 : for (unsigned int i = 0; i != vec_oprnds0.length() / 2; i++)
5883 : {
5884 : /* Arguments are ready, create the new vector stmt. */
5885 8 : gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
5886 8 : gimple *new_stmt
5887 8 : = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
5888 8 : new_temp = make_ssa_name (vec_dest, new_stmt);
5889 8 : gimple_set_lhs (new_stmt, new_temp);
5890 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5891 :
5892 : /* This is the last step of the conversion sequence. Store the
5893 : vectors in SLP_NODE or in vector info of the scalar statement
5894 : (or in STMT_VINFO_RELATED_STMT chain). */
5895 8 : slp_node->push_vec_def (new_stmt);
5896 : }
5897 : }
5898 : break;
5899 : }
5900 :
5901 23004 : vec_oprnds0.release ();
5902 23004 : vec_oprnds1.release ();
5903 23004 : interm_types.release ();
5904 :
5905 23004 : return true;
5906 181340 : }
5907 :
5908 : /* Return true if we can assume from the scalar form of STMT_INFO that
5909 : neither the scalar nor the vector forms will generate code. STMT_INFO
5910 : is known not to involve a data reference. */
5911 :
5912 : bool
5913 3205103 : vect_nop_conversion_p (stmt_vec_info stmt_info)
5914 : {
5915 3205103 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5916 2924769 : if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
5917 : return false;
5918 :
5919 937856 : tree lhs = gimple_assign_lhs (stmt);
5920 937856 : tree_code code = gimple_assign_rhs_code (stmt);
5921 937856 : tree rhs = gimple_assign_rhs1 (stmt);
5922 :
5923 937856 : if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
5924 : return true;
5925 :
5926 934824 : if (CONVERT_EXPR_CODE_P (code))
5927 234084 : return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
5928 :
5929 : return false;
5930 : }
5931 :
5932 : /* Function vectorizable_assignment.
5933 :
5934 : Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5935 : If COST_VEC is passed, calculate costs but don't change anything,
5936 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5937 : it, and insert it at GSI.
5938 : Return true if STMT_INFO is vectorizable in this way. */
5939 :
5940 : static bool
5941 2097476 : vectorizable_assignment (vec_info *vinfo,
5942 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5943 : slp_tree slp_node,
5944 : stmt_vector_for_cost *cost_vec)
5945 : {
5946 2097476 : tree vec_dest;
5947 2097476 : tree scalar_dest;
5948 2097476 : tree op;
5949 2097476 : tree new_temp;
5950 2097476 : enum vect_def_type dt[1] = {vect_unknown_def_type};
5951 2097476 : int i;
5952 2097476 : vec<tree> vec_oprnds = vNULL;
5953 2097476 : tree vop;
5954 2097476 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5955 2097476 : enum tree_code code;
5956 2097476 : tree vectype_in;
5957 :
5958 2097476 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5959 : return false;
5960 :
5961 2097476 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5962 237977 : && cost_vec)
5963 : return false;
5964 :
5965 : /* Is vectorizable assignment? */
5966 3797302 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5967 1786023 : if (!stmt)
5968 : return false;
5969 :
5970 1786023 : scalar_dest = gimple_assign_lhs (stmt);
5971 1786023 : if (TREE_CODE (scalar_dest) != SSA_NAME)
5972 : return false;
5973 :
5974 960843 : if (STMT_VINFO_DATA_REF (stmt_info))
5975 : return false;
5976 :
5977 405776 : code = gimple_assign_rhs_code (stmt);
5978 405776 : if (!(gimple_assign_single_p (stmt)
5979 404163 : || code == PAREN_EXPR
5980 402986 : || CONVERT_EXPR_CODE_P (code)))
5981 : return false;
5982 :
5983 97415 : tree vectype = SLP_TREE_VECTYPE (slp_node);
5984 97415 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
5985 :
5986 97415 : slp_tree slp_op;
5987 97415 : if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
5988 : &dt[0], &vectype_in))
5989 : {
5990 0 : if (dump_enabled_p ())
5991 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5992 : "use not simple.\n");
5993 0 : return false;
5994 : }
5995 97415 : if (!vectype_in)
5996 18639 : vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
5997 :
5998 : /* We can handle VIEW_CONVERT conversions that do not change the number
5999 : of elements or the vector size or other conversions when the component
6000 : types are nop-convertible. */
6001 97415 : if (!vectype_in
6002 97135 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
6003 89604 : || (code == VIEW_CONVERT_EXPR
6004 2970 : && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
6005 2970 : GET_MODE_SIZE (TYPE_MODE (vectype_in))))
6006 187019 : || (CONVERT_EXPR_CODE_P (code)
6007 86846 : && !tree_nop_conversion_p (TREE_TYPE (vectype),
6008 86846 : TREE_TYPE (vectype_in))))
6009 10918 : return false;
6010 :
6011 259083 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
6012 : {
6013 26 : if (dump_enabled_p ())
6014 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6015 : "can't convert between boolean and non "
6016 0 : "boolean vectors %T\n", TREE_TYPE (op));
6017 :
6018 26 : return false;
6019 : }
6020 :
6021 : /* We do not handle bit-precision changes. */
6022 86471 : if ((CONVERT_EXPR_CODE_P (code)
6023 2758 : || code == VIEW_CONVERT_EXPR)
6024 85198 : && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6025 83850 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6026 84731 : || (INTEGRAL_TYPE_P (TREE_TYPE (op))
6027 79535 : && !type_has_mode_precision_p (TREE_TYPE (op))))
6028 : /* But a conversion that does not change the bit-pattern is ok. */
6029 87429 : && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6030 958 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
6031 958 : && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
6032 958 : > TYPE_PRECISION (TREE_TYPE (op)))
6033 491 : && TYPE_UNSIGNED (TREE_TYPE (op)))
6034 483 : || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
6035 483 : == TYPE_PRECISION (TREE_TYPE (op))))))
6036 : {
6037 274 : if (dump_enabled_p ())
6038 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6039 : "type conversion to/from bit-precision "
6040 : "unsupported.\n");
6041 274 : return false;
6042 : }
6043 :
6044 86197 : if (cost_vec) /* transformation not required. */
6045 : {
6046 70091 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
6047 : {
6048 0 : if (dump_enabled_p ())
6049 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6050 : "incompatible vector types for invariants\n");
6051 0 : return false;
6052 : }
6053 70091 : SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
6054 70091 : DUMP_VECT_SCOPE ("vectorizable_assignment");
6055 70091 : if (!vect_nop_conversion_p (stmt_info))
6056 979 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6057 70091 : return true;
6058 : }
6059 :
6060 : /* Transform. */
6061 16106 : if (dump_enabled_p ())
6062 3625 : dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
6063 :
6064 : /* Handle def. */
6065 16106 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6066 :
6067 : /* Handle use. */
6068 16106 : vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
6069 :
6070 : /* Arguments are ready. create the new vector stmt. */
6071 36361 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
6072 : {
6073 20255 : if (CONVERT_EXPR_CODE_P (code)
6074 703 : || code == VIEW_CONVERT_EXPR)
6075 19710 : vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
6076 20255 : gassign *new_stmt = gimple_build_assign (vec_dest, vop);
6077 20255 : new_temp = make_ssa_name (vec_dest, new_stmt);
6078 20255 : gimple_assign_set_lhs (new_stmt, new_temp);
6079 20255 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6080 20255 : slp_node->push_vec_def (new_stmt);
6081 : }
6082 :
6083 16106 : vec_oprnds.release ();
6084 16106 : return true;
6085 : }
6086 :
6087 :
6088 : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6089 : either as shift by a scalar or by a vector. */
6090 :
6091 : bool
6092 301696 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
6093 : {
6094 301696 : optab optab;
6095 301696 : tree vectype;
6096 :
6097 301696 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6098 301696 : if (!vectype)
6099 : return false;
6100 :
6101 301696 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6102 301696 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6103 : return true;
6104 :
6105 265153 : optab = optab_for_tree_code (code, vectype, optab_vector);
6106 265153 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6107 : return true;
6108 :
6109 : return false;
6110 : }
6111 :
6112 :
6113 : /* Function vectorizable_shift.
6114 :
6115 : Check if STMT_INFO performs a shift operation that can be vectorized.
6116 : If COST_VEC is passed, calculate costs but don't change anything,
6117 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6118 : it, and insert it at GSI.
6119 : Return true if STMT_INFO is vectorizable in this way. */
6120 :
6121 : static bool
6122 748435 : vectorizable_shift (vec_info *vinfo,
6123 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6124 : slp_tree slp_node,
6125 : stmt_vector_for_cost *cost_vec)
6126 : {
6127 748435 : tree vec_dest;
6128 748435 : tree scalar_dest;
6129 748435 : tree op0, op1 = NULL;
6130 748435 : tree vec_oprnd1 = NULL_TREE;
6131 748435 : tree vectype;
6132 748435 : enum tree_code code;
6133 748435 : machine_mode vec_mode;
6134 748435 : tree new_temp;
6135 748435 : optab optab;
6136 748435 : int icode;
6137 748435 : machine_mode optab_op2_mode;
6138 748435 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
6139 748435 : poly_uint64 nunits_in;
6140 748435 : poly_uint64 nunits_out;
6141 748435 : tree vectype_out;
6142 748435 : tree op1_vectype;
6143 748435 : int i;
6144 748435 : vec<tree> vec_oprnds0 = vNULL;
6145 748435 : vec<tree> vec_oprnds1 = vNULL;
6146 748435 : tree vop0, vop1;
6147 748435 : unsigned int k;
6148 748435 : bool scalar_shift_arg = true;
6149 748435 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6150 748435 : bool incompatible_op1_vectype_p = false;
6151 :
6152 748435 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6153 : return false;
6154 :
6155 748435 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6156 237977 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
6157 236487 : && cost_vec)
6158 : return false;
6159 :
6160 : /* Is STMT a vectorizable binary/unary operation? */
6161 1129115 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6162 439463 : if (!stmt)
6163 : return false;
6164 :
6165 439463 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
6166 : return false;
6167 :
6168 438905 : code = gimple_assign_rhs_code (stmt);
6169 :
6170 438905 : if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
6171 : || code == RROTATE_EXPR))
6172 : return false;
6173 :
6174 64905 : scalar_dest = gimple_assign_lhs (stmt);
6175 64905 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6176 64905 : if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6177 : {
6178 0 : if (dump_enabled_p ())
6179 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6180 : "bit-precision shifts not supported.\n");
6181 0 : return false;
6182 : }
6183 :
6184 64905 : slp_tree slp_op0;
6185 64905 : if (!vect_is_simple_use (vinfo, slp_node,
6186 : 0, &op0, &slp_op0, &dt[0], &vectype))
6187 : {
6188 0 : if (dump_enabled_p ())
6189 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6190 : "use not simple.\n");
6191 0 : return false;
6192 : }
6193 : /* If op0 is an external or constant def, infer the vector type
6194 : from the scalar type. */
6195 64905 : if (!vectype)
6196 15492 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
6197 64905 : if (!cost_vec)
6198 8676 : gcc_assert (vectype);
6199 64905 : if (!vectype)
6200 : {
6201 0 : if (dump_enabled_p ())
6202 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6203 : "no vectype for scalar type\n");
6204 0 : return false;
6205 : }
6206 :
6207 64905 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6208 64905 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6209 64905 : if (maybe_ne (nunits_out, nunits_in))
6210 : return false;
6211 :
6212 64905 : stmt_vec_info op1_def_stmt_info;
6213 64905 : slp_tree slp_op1;
6214 64905 : if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
6215 : &dt[1], &op1_vectype, &op1_def_stmt_info))
6216 : {
6217 0 : if (dump_enabled_p ())
6218 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6219 : "use not simple.\n");
6220 0 : return false;
6221 : }
6222 :
6223 : /* Determine whether the shift amount is a vector, or scalar. If the
6224 : shift/rotate amount is a vector, use the vector/vector shift optabs. */
6225 :
6226 64905 : if ((dt[1] == vect_internal_def
6227 64905 : || dt[1] == vect_induction_def
6228 48580 : || dt[1] == vect_nested_cycle)
6229 16343 : && SLP_TREE_LANES (slp_node) == 1)
6230 : scalar_shift_arg = false;
6231 48617 : else if (dt[1] == vect_constant_def
6232 : || dt[1] == vect_external_def
6233 48617 : || dt[1] == vect_internal_def)
6234 : {
6235 : /* In SLP, need to check whether the shift count is the same,
6236 : in loops if it is a constant or invariant, it is always
6237 : a scalar shift. */
6238 48611 : vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
6239 48611 : stmt_vec_info slpstmt_info;
6240 :
6241 128640 : FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
6242 80029 : if (slpstmt_info)
6243 : {
6244 80029 : gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
6245 160058 : if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
6246 80029 : scalar_shift_arg = false;
6247 : }
6248 :
6249 : /* For internal SLP defs we have to make sure we see scalar stmts
6250 : for all vector elements.
6251 : ??? For different vectors we could resort to a different
6252 : scalar shift operand but code-generation below simply always
6253 : takes the first. */
6254 48611 : if (dt[1] == vect_internal_def
6255 48660 : && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
6256 49 : stmts.length ()))
6257 : scalar_shift_arg = false;
6258 :
6259 : /* If the shift amount is computed by a pattern stmt we cannot
6260 : use the scalar amount directly thus give up and use a vector
6261 : shift. */
6262 48611 : if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
6263 : scalar_shift_arg = false;
6264 : }
6265 : else
6266 : {
6267 6 : if (dump_enabled_p ())
6268 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6269 : "operand mode requires invariant argument.\n");
6270 6 : return false;
6271 : }
6272 :
6273 : /* Vector shifted by vector. */
6274 64937 : bool was_scalar_shift_arg = scalar_shift_arg;
6275 48602 : if (!scalar_shift_arg)
6276 : {
6277 16335 : optab = optab_for_tree_code (code, vectype, optab_vector);
6278 16335 : if (dump_enabled_p ())
6279 1205 : dump_printf_loc (MSG_NOTE, vect_location,
6280 : "vector/vector shift/rotate found.\n");
6281 :
6282 16335 : if (!op1_vectype)
6283 15 : op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
6284 : slp_op1);
6285 16335 : incompatible_op1_vectype_p
6286 32670 : = (op1_vectype == NULL_TREE
6287 16335 : || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
6288 16335 : TYPE_VECTOR_SUBPARTS (vectype))
6289 32668 : || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
6290 16328 : if (incompatible_op1_vectype_p
6291 7 : && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
6292 1 : || slp_op1->refcnt != 1))
6293 : {
6294 6 : if (dump_enabled_p ())
6295 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6296 : "unusable type for last operand in"
6297 : " vector/vector shift/rotate.\n");
6298 6 : return false;
6299 : }
6300 : }
6301 : /* See if the machine has a vector shifted by scalar insn and if not
6302 : then see if it has a vector shifted by vector insn. */
6303 : else
6304 : {
6305 48564 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6306 48564 : if (optab
6307 48564 : && can_implement_p (optab, TYPE_MODE (vectype)))
6308 : {
6309 48564 : if (dump_enabled_p ())
6310 4946 : dump_printf_loc (MSG_NOTE, vect_location,
6311 : "vector/scalar shift/rotate found.\n");
6312 : }
6313 : else
6314 : {
6315 0 : optab = optab_for_tree_code (code, vectype, optab_vector);
6316 0 : if (optab
6317 0 : && can_implement_p (optab, TYPE_MODE (vectype)))
6318 : {
6319 0 : scalar_shift_arg = false;
6320 :
6321 0 : if (dump_enabled_p ())
6322 0 : dump_printf_loc (MSG_NOTE, vect_location,
6323 : "vector/vector shift/rotate found.\n");
6324 :
6325 0 : if (!op1_vectype)
6326 0 : op1_vectype = get_vectype_for_scalar_type (vinfo,
6327 0 : TREE_TYPE (op1),
6328 : slp_op1);
6329 :
6330 : /* Unlike the other binary operators, shifts/rotates have
6331 : the rhs being int, instead of the same type as the lhs,
6332 : so make sure the scalar is the right type if we are
6333 : dealing with vectors of long long/long/short/char. */
6334 0 : incompatible_op1_vectype_p
6335 0 : = (!op1_vectype
6336 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype),
6337 0 : TREE_TYPE (op1)));
6338 0 : if (incompatible_op1_vectype_p
6339 0 : && dt[1] == vect_internal_def)
6340 : {
6341 0 : if (dump_enabled_p ())
6342 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6343 : "unusable type for last operand in"
6344 : " vector/vector shift/rotate.\n");
6345 0 : return false;
6346 : }
6347 : }
6348 : }
6349 : }
6350 :
6351 : /* Supportable by target? */
6352 64893 : if (!optab)
6353 : {
6354 0 : if (dump_enabled_p ())
6355 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6356 : "no shift optab for %s and %T.\n",
6357 : get_tree_code_name (code), vectype);
6358 0 : return false;
6359 : }
6360 64893 : vec_mode = TYPE_MODE (vectype);
6361 64893 : icode = (int) optab_handler (optab, vec_mode);
6362 64893 : if (icode == CODE_FOR_nothing)
6363 : {
6364 6110 : if (dump_enabled_p ())
6365 900 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6366 : "shift op not supported by target.\n");
6367 6110 : return false;
6368 : }
6369 : /* vector lowering cannot optimize vector shifts using word arithmetic. */
6370 58783 : if (vect_emulated_vector_p (vectype))
6371 : return false;
6372 :
6373 58783 : if (cost_vec) /* transformation not required. */
6374 : {
6375 50107 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6376 50107 : || ((!scalar_shift_arg || dt[1] == vect_internal_def)
6377 8077 : && (!incompatible_op1_vectype_p
6378 1 : || dt[1] == vect_constant_def)
6379 8077 : && !vect_maybe_update_slp_op_vectype
6380 8077 : (slp_op1,
6381 : incompatible_op1_vectype_p ? vectype : op1_vectype)))
6382 : {
6383 0 : if (dump_enabled_p ())
6384 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6385 : "incompatible vector types for invariants\n");
6386 0 : return false;
6387 : }
6388 : /* Now adjust the constant shift amount in place. */
6389 50107 : if (incompatible_op1_vectype_p
6390 1 : && dt[1] == vect_constant_def)
6391 4 : for (unsigned i = 0;
6392 5 : i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
6393 : {
6394 4 : SLP_TREE_SCALAR_OPS (slp_op1)[i]
6395 4 : = fold_convert (TREE_TYPE (vectype),
6396 : SLP_TREE_SCALAR_OPS (slp_op1)[i]);
6397 4 : gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
6398 : == INTEGER_CST));
6399 : }
6400 50107 : SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
6401 50107 : DUMP_VECT_SCOPE ("vectorizable_shift");
6402 50107 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6403 50107 : return true;
6404 : }
6405 :
6406 : /* Transform. */
6407 :
6408 8676 : if (dump_enabled_p ())
6409 2033 : dump_printf_loc (MSG_NOTE, vect_location,
6410 : "transform binary/unary operation.\n");
6411 :
6412 : /* Handle def. */
6413 8676 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6414 :
6415 8676 : unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
6416 8676 : if (scalar_shift_arg && dt[1] != vect_internal_def)
6417 : {
6418 : /* Vector shl and shr insn patterns can be defined with scalar
6419 : operand 2 (shift operand). In this case, use constant or loop
6420 : invariant op1 directly, without extending it to vector mode
6421 : first. */
6422 6514 : optab_op2_mode = insn_data[icode].operand[2].mode;
6423 6514 : if (!VECTOR_MODE_P (optab_op2_mode))
6424 : {
6425 6514 : if (dump_enabled_p ())
6426 1918 : dump_printf_loc (MSG_NOTE, vect_location,
6427 : "operand 1 using scalar mode.\n");
6428 6514 : vec_oprnd1 = op1;
6429 6514 : vec_oprnds1.create (nvectors);
6430 6514 : vec_oprnds1.quick_push (vec_oprnd1);
6431 : /* Store vec_oprnd1 for every vector stmt to be created.
6432 : We check during the analysis that all the shift arguments
6433 : are the same.
6434 : TODO: Allow different constants for different vector
6435 : stmts generated for an SLP instance. */
6436 15117 : for (k = 0; k < nvectors - 1; k++)
6437 2089 : vec_oprnds1.quick_push (vec_oprnd1);
6438 : }
6439 : }
6440 2162 : else if (!scalar_shift_arg && incompatible_op1_vectype_p)
6441 : {
6442 0 : if (was_scalar_shift_arg)
6443 : {
6444 : /* If the argument was the same in all lanes create the
6445 : correctly typed vector shift amount directly. Note
6446 : we made SLP scheduling think we use the original scalars,
6447 : so place the compensation code next to the shift which
6448 : is conservative. See PR119640 where it otherwise breaks. */
6449 0 : op1 = fold_convert (TREE_TYPE (vectype), op1);
6450 0 : op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
6451 : gsi);
6452 0 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
6453 : gsi);
6454 0 : vec_oprnds1.create (nvectors);
6455 0 : for (k = 0; k < nvectors; k++)
6456 0 : vec_oprnds1.quick_push (vec_oprnd1);
6457 : }
6458 0 : else if (dt[1] == vect_constant_def)
6459 : /* The constant shift amount has been adjusted in place. */
6460 : ;
6461 : else
6462 0 : gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
6463 : }
6464 :
6465 : /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6466 : (a special case for certain kind of vector shifts); otherwise,
6467 : operand 1 should be of a vector type (the usual case). */
6468 2162 : vect_get_vec_defs (vinfo, slp_node,
6469 : op0, &vec_oprnds0,
6470 8676 : vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1);
6471 :
6472 : /* Arguments are ready. Create the new vector stmt. */
6473 22943 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6474 : {
6475 : /* For internal defs where we need to use a scalar shift arg
6476 : extract the first lane. */
6477 14267 : if (scalar_shift_arg && dt[1] == vect_internal_def)
6478 : {
6479 10 : vop1 = vec_oprnds1[0];
6480 10 : new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
6481 10 : gassign *new_stmt
6482 10 : = gimple_build_assign (new_temp,
6483 10 : build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
6484 : vop1,
6485 10 : TYPE_SIZE (TREE_TYPE (new_temp)),
6486 : bitsize_zero_node));
6487 10 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6488 10 : vop1 = new_temp;
6489 10 : }
6490 : else
6491 14257 : vop1 = vec_oprnds1[i];
6492 14267 : gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
6493 14267 : new_temp = make_ssa_name (vec_dest, new_stmt);
6494 14267 : gimple_assign_set_lhs (new_stmt, new_temp);
6495 14267 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6496 14267 : slp_node->push_vec_def (new_stmt);
6497 : }
6498 :
6499 8676 : vec_oprnds0.release ();
6500 8676 : vec_oprnds1.release ();
6501 :
6502 8676 : return true;
6503 : }
6504 :
6505 : /* Function vectorizable_operation.
6506 :
6507 : Check if STMT_INFO performs a binary, unary or ternary operation that can
6508 : be vectorized.
6509 : If COST_VEC is passed, calculate costs but don't change anything,
6510 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6511 : it, and insert it at GSI.
6512 : Return true if STMT_INFO is vectorizable in this way. */
6513 :
6514 : static bool
6515 2737931 : vectorizable_operation (vec_info *vinfo,
6516 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6517 : slp_tree slp_node,
6518 : stmt_vector_for_cost *cost_vec)
6519 : {
6520 2737931 : tree vec_dest;
6521 2737931 : tree scalar_dest;
6522 2737931 : tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
6523 2737931 : tree vectype;
6524 2737931 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6525 2737931 : enum tree_code code, orig_code;
6526 2737931 : machine_mode vec_mode;
6527 2737931 : tree new_temp;
6528 2737931 : int op_type;
6529 2737931 : optab optab;
6530 2737931 : bool target_support_p;
6531 2737931 : enum vect_def_type dt[3]
6532 : = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
6533 2737931 : poly_uint64 nunits_in;
6534 2737931 : poly_uint64 nunits_out;
6535 2737931 : tree vectype_out;
6536 2737931 : int i;
6537 2737931 : vec<tree> vec_oprnds0 = vNULL;
6538 2737931 : vec<tree> vec_oprnds1 = vNULL;
6539 2737931 : vec<tree> vec_oprnds2 = vNULL;
6540 2737931 : tree vop0, vop1, vop2;
6541 2737931 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6542 :
6543 2737931 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6544 : return false;
6545 :
6546 2737931 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6547 237977 : && cost_vec)
6548 : return false;
6549 :
6550 : /* Is STMT a vectorizable binary/unary operation? */
6551 4507848 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6552 2426478 : if (!stmt)
6553 : return false;
6554 :
6555 : /* Loads and stores are handled in vectorizable_{load,store}. */
6556 2426478 : if (STMT_VINFO_DATA_REF (stmt_info))
6557 : return false;
6558 :
6559 1046231 : orig_code = code = gimple_assign_rhs_code (stmt);
6560 :
6561 : /* Shifts are handled in vectorizable_shift. */
6562 1046231 : if (code == LSHIFT_EXPR
6563 : || code == RSHIFT_EXPR
6564 : || code == LROTATE_EXPR
6565 1046231 : || code == RROTATE_EXPR)
6566 : return false;
6567 :
6568 : /* Comparisons are handled in vectorizable_comparison. */
6569 990002 : if (TREE_CODE_CLASS (code) == tcc_comparison)
6570 : return false;
6571 :
6572 : /* Conditions are handled in vectorizable_condition. */
6573 794949 : if (code == COND_EXPR)
6574 : return false;
6575 :
6576 : /* For pointer addition and subtraction, we should use the normal
6577 : plus and minus for the vector operation. */
6578 768515 : if (code == POINTER_PLUS_EXPR)
6579 : code = PLUS_EXPR;
6580 748639 : if (code == POINTER_DIFF_EXPR)
6581 949 : code = MINUS_EXPR;
6582 :
6583 : /* Support only unary or binary operations. */
6584 768515 : op_type = TREE_CODE_LENGTH (code);
6585 768515 : if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
6586 : {
6587 0 : if (dump_enabled_p ())
6588 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6589 : "num. args = %d (not unary/binary/ternary op).\n",
6590 : op_type);
6591 0 : return false;
6592 : }
6593 :
6594 768515 : scalar_dest = gimple_assign_lhs (stmt);
6595 768515 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6596 :
6597 : /* Most operations cannot handle bit-precision types without extra
6598 : truncations. */
6599 768515 : bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
6600 757191 : if (!mask_op_p
6601 757191 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
6602 : /* Exception are bitwise binary operations. */
6603 : && code != BIT_IOR_EXPR
6604 1812 : && code != BIT_XOR_EXPR
6605 1058 : && code != BIT_AND_EXPR)
6606 : {
6607 746 : if (dump_enabled_p ())
6608 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6609 : "bit-precision arithmetic not supported.\n");
6610 746 : return false;
6611 : }
6612 :
6613 767769 : slp_tree slp_op0;
6614 767769 : if (!vect_is_simple_use (vinfo, slp_node,
6615 : 0, &op0, &slp_op0, &dt[0], &vectype))
6616 : {
6617 0 : if (dump_enabled_p ())
6618 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6619 : "use not simple.\n");
6620 0 : return false;
6621 : }
6622 767769 : bool is_invariant = (dt[0] == vect_external_def
6623 767769 : || dt[0] == vect_constant_def);
6624 : /* If op0 is an external or constant def, infer the vector type
6625 : from the scalar type. */
6626 767769 : if (!vectype)
6627 : {
6628 : /* For boolean type we cannot determine vectype by
6629 : invariant value (don't know whether it is a vector
6630 : of booleans or vector of integers). We use output
6631 : vectype because operations on boolean don't change
6632 : type. */
6633 70603 : if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
6634 : {
6635 1876 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
6636 : {
6637 274 : if (dump_enabled_p ())
6638 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6639 : "not supported operation on bool value.\n");
6640 274 : return false;
6641 : }
6642 1602 : vectype = vectype_out;
6643 : }
6644 : else
6645 68727 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
6646 : slp_node);
6647 : }
6648 767495 : if (!cost_vec)
6649 114895 : gcc_assert (vectype);
6650 767495 : if (!vectype)
6651 : {
6652 292 : if (dump_enabled_p ())
6653 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6654 : "no vectype for scalar type %T\n",
6655 2 : TREE_TYPE (op0));
6656 :
6657 292 : return false;
6658 : }
6659 :
6660 767203 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6661 767203 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6662 767203 : if (maybe_ne (nunits_out, nunits_in)
6663 767203 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
6664 12373 : return false;
6665 :
6666 754830 : tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
6667 754830 : slp_tree slp_op1 = NULL, slp_op2 = NULL;
6668 754830 : if (op_type == binary_op || op_type == ternary_op)
6669 : {
6670 674420 : if (!vect_is_simple_use (vinfo, slp_node,
6671 : 1, &op1, &slp_op1, &dt[1], &vectype2))
6672 : {
6673 0 : if (dump_enabled_p ())
6674 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6675 : "use not simple.\n");
6676 0 : return false;
6677 : }
6678 674420 : is_invariant &= (dt[1] == vect_external_def
6679 674420 : || dt[1] == vect_constant_def);
6680 674420 : if (vectype2
6681 1143056 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
6682 468636 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6683 468636 : TREE_TYPE (vectype2))))
6684 4 : return false;
6685 : }
6686 754826 : if (op_type == ternary_op)
6687 : {
6688 0 : if (!vect_is_simple_use (vinfo, slp_node,
6689 : 2, &op2, &slp_op2, &dt[2], &vectype3))
6690 : {
6691 0 : if (dump_enabled_p ())
6692 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6693 : "use not simple.\n");
6694 0 : return false;
6695 : }
6696 0 : is_invariant &= (dt[2] == vect_external_def
6697 0 : || dt[2] == vect_constant_def);
6698 0 : if (vectype3
6699 0 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
6700 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6701 0 : TREE_TYPE (vectype3))))
6702 0 : return false;
6703 : }
6704 :
6705 : /* Multiple types in SLP are handled by creating the appropriate number of
6706 : vectorized stmts for each SLP node. */
6707 754826 : auto vec_num = vect_get_num_copies (vinfo, slp_node);
6708 :
6709 : /* Reject attempts to combine mask types with nonmask types, e.g. if
6710 : we have an AND between a (nonmask) boolean loaded from memory and
6711 : a (mask) boolean result of a comparison.
6712 :
6713 : TODO: We could easily fix these cases up using pattern statements. */
6714 754826 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
6715 1215414 : || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
6716 1509652 : || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
6717 : {
6718 0 : if (dump_enabled_p ())
6719 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6720 : "mixed mask and nonmask vector types\n");
6721 0 : return false;
6722 : }
6723 :
6724 : /* Supportable by target? */
6725 :
6726 754826 : vec_mode = TYPE_MODE (vectype);
6727 754826 : optab = optab_for_tree_code (code, vectype, optab_default);
6728 754826 : if (!optab)
6729 : {
6730 69333 : if (dump_enabled_p ())
6731 5961 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6732 : "no optab for %s and %T.\n",
6733 : get_tree_code_name (code), vectype);
6734 69333 : return false;
6735 : }
6736 685493 : target_support_p = can_implement_p (optab, vec_mode);
6737 :
6738 685493 : bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
6739 685493 : if (!target_support_p || using_emulated_vectors_p)
6740 : {
6741 29397 : if (dump_enabled_p ())
6742 1128 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6743 : "op not supported by target.\n");
6744 : /* When vec_mode is not a vector mode and we verified ops we
6745 : do not have to lower like AND are natively supported let
6746 : those through even when the mode isn't word_mode. For
6747 : ops we have to lower the lowering code assumes we are
6748 : dealing with word_mode. */
6749 58794 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
6750 29255 : || !GET_MODE_SIZE (vec_mode).is_constant ()
6751 29255 : || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6752 24179 : || !target_support_p)
6753 62528 : && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
6754 : /* Check only during analysis. */
6755 41267 : || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
6756 : {
6757 28799 : if (dump_enabled_p ())
6758 1126 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
6759 28799 : return false;
6760 : }
6761 598 : if (dump_enabled_p ())
6762 2 : dump_printf_loc (MSG_NOTE, vect_location,
6763 : "proceeding using word mode.\n");
6764 : using_emulated_vectors_p = true;
6765 : }
6766 :
6767 656694 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
6768 656694 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
6769 435301 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
6770 656694 : internal_fn cond_fn = get_conditional_internal_fn (code);
6771 656694 : internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
6772 :
6773 : /* If operating on inactive elements could generate spurious traps,
6774 : we need to restrict the operation to active lanes. Note that this
6775 : specifically doesn't apply to unhoisted invariants, since they
6776 : operate on the same value for every lane.
6777 :
6778 : Similarly, if this operation is part of a reduction, a fully-masked
6779 : loop should only change the active lanes of the reduction chain,
6780 : keeping the inactive lanes as-is. */
6781 628362 : bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
6782 1221225 : || reduc_idx >= 0);
6783 :
6784 656694 : if (cost_vec) /* transformation not required. */
6785 : {
6786 541799 : if (loop_vinfo
6787 331987 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
6788 89906 : && mask_out_inactive)
6789 : {
6790 20416 : if (cond_len_fn != IFN_LAST
6791 20416 : && direct_internal_fn_supported_p (cond_len_fn, vectype,
6792 : OPTIMIZE_FOR_SPEED))
6793 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
6794 : 1);
6795 20416 : else if (cond_fn != IFN_LAST
6796 20416 : && direct_internal_fn_supported_p (cond_fn, vectype,
6797 : OPTIMIZE_FOR_SPEED))
6798 8514 : vect_record_loop_mask (loop_vinfo, masks, vec_num,
6799 : vectype, NULL);
6800 : else
6801 : {
6802 11902 : if (dump_enabled_p ())
6803 610 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6804 : "can't use a fully-masked loop because no"
6805 : " conditional operation is available.\n");
6806 11902 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6807 : }
6808 : }
6809 :
6810 : /* Put types on constant and invariant SLP children. */
6811 541799 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6812 541752 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
6813 1083465 : || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
6814 : {
6815 133 : if (dump_enabled_p ())
6816 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6817 : "incompatible vector types for invariants\n");
6818 133 : return false;
6819 : }
6820 :
6821 541666 : SLP_TREE_TYPE (slp_node) = op_vec_info_type;
6822 541666 : DUMP_VECT_SCOPE ("vectorizable_operation");
6823 541666 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6824 541666 : if (using_emulated_vectors_p)
6825 : {
6826 : /* The above vect_model_simple_cost call handles constants
6827 : in the prologue and (mis-)costs one of the stmts as
6828 : vector stmt. See below for the actual lowering that will
6829 : be applied. */
6830 596 : unsigned n = vect_get_num_copies (vinfo, slp_node);
6831 596 : switch (code)
6832 : {
6833 213 : case PLUS_EXPR:
6834 213 : n *= 5;
6835 213 : break;
6836 352 : case MINUS_EXPR:
6837 352 : n *= 6;
6838 352 : break;
6839 0 : case NEGATE_EXPR:
6840 0 : n *= 4;
6841 0 : break;
6842 : default:
6843 : /* Bit operations do not have extra cost and are accounted
6844 : as vector stmt by vect_model_simple_cost. */
6845 : n = 0;
6846 : break;
6847 : }
6848 565 : if (n != 0)
6849 : {
6850 : /* We also need to materialize two large constants. */
6851 565 : record_stmt_cost (cost_vec, 2, scalar_stmt, stmt_info,
6852 : 0, vect_prologue);
6853 565 : record_stmt_cost (cost_vec, n, scalar_stmt, stmt_info,
6854 : 0, vect_body);
6855 : }
6856 : }
6857 541666 : return true;
6858 : }
6859 :
6860 : /* Transform. */
6861 :
6862 114895 : if (dump_enabled_p ())
6863 16478 : dump_printf_loc (MSG_NOTE, vect_location,
6864 : "transform binary/unary operation.\n");
6865 :
6866 114895 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
6867 103314 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
6868 :
6869 : /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6870 : vectors with unsigned elements, but the result is signed. So, we
6871 : need to compute the MINUS_EXPR into vectype temporary and
6872 : VIEW_CONVERT_EXPR it into the final vectype_out result. */
6873 114895 : tree vec_cvt_dest = NULL_TREE;
6874 114895 : if (orig_code == POINTER_DIFF_EXPR)
6875 : {
6876 113 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6877 113 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6878 : }
6879 : /* For reduction operations with undefined overflow behavior make sure to
6880 : pun them to unsigned since we change the order of evaluation.
6881 : ??? Avoid for in-order reductions? */
6882 114782 : else if (arith_code_with_undefined_signed_overflow (orig_code)
6883 98052 : && ANY_INTEGRAL_TYPE_P (vectype)
6884 47849 : && TYPE_OVERFLOW_UNDEFINED (vectype)
6885 140435 : && SLP_TREE_REDUC_IDX (slp_node) != -1)
6886 : {
6887 2471 : gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
6888 : || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
6889 2471 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6890 2471 : vectype = unsigned_type_for (vectype);
6891 2471 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6892 : }
6893 : /* Handle def. */
6894 : else
6895 112311 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
6896 :
6897 114895 : vect_get_vec_defs (vinfo, slp_node,
6898 : op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2);
6899 : /* Arguments are ready. Create the new vector stmt. */
6900 254096 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6901 : {
6902 139201 : gimple *new_stmt = NULL;
6903 278402 : vop1 = ((op_type == binary_op || op_type == ternary_op)
6904 139201 : ? vec_oprnds1[i] : NULL_TREE);
6905 139201 : vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
6906 :
6907 139201 : if (vec_cvt_dest
6908 139201 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
6909 : {
6910 2924 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
6911 2924 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6912 : new_temp);
6913 2924 : new_temp = make_ssa_name (vec_dest, new_stmt);
6914 2924 : gimple_assign_set_lhs (new_stmt, new_temp);
6915 2924 : vect_finish_stmt_generation (vinfo, stmt_info,
6916 : new_stmt, gsi);
6917 2924 : vop0 = new_temp;
6918 : }
6919 139201 : if (vop1
6920 136573 : && vec_cvt_dest
6921 142253 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
6922 : {
6923 2924 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
6924 2924 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6925 : new_temp);
6926 2924 : new_temp = make_ssa_name (vec_dest, new_stmt);
6927 2924 : gimple_assign_set_lhs (new_stmt, new_temp);
6928 2924 : vect_finish_stmt_generation (vinfo, stmt_info,
6929 : new_stmt, gsi);
6930 2924 : vop1 = new_temp;
6931 : }
6932 139201 : if (vop2
6933 0 : && vec_cvt_dest
6934 139201 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
6935 : {
6936 0 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
6937 0 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6938 : new_temp);
6939 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6940 0 : gimple_assign_set_lhs (new_stmt, new_temp);
6941 0 : vect_finish_stmt_generation (vinfo, stmt_info,
6942 : new_stmt, gsi);
6943 0 : vop2 = new_temp;
6944 : }
6945 :
6946 139201 : if (using_emulated_vectors_p)
6947 : {
6948 : /* Lower the operation. This follows vector lowering. */
6949 2 : tree word_type = build_nonstandard_integer_type
6950 2 : (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
6951 2 : tree wvop0 = make_ssa_name (word_type);
6952 2 : new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
6953 : build1 (VIEW_CONVERT_EXPR,
6954 : word_type, vop0));
6955 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6956 2 : tree wvop1 = NULL_TREE;
6957 2 : if (vop1)
6958 : {
6959 2 : wvop1 = make_ssa_name (word_type);
6960 2 : new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
6961 : build1 (VIEW_CONVERT_EXPR,
6962 : word_type, vop1));
6963 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6964 : }
6965 :
6966 2 : tree result_low;
6967 2 : if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6968 : {
6969 1 : unsigned int width = vector_element_bits (vectype);
6970 1 : tree inner_type = TREE_TYPE (vectype);
6971 1 : HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
6972 1 : tree low_bits
6973 1 : = build_replicated_int_cst (word_type, width, max >> 1);
6974 1 : tree high_bits
6975 2 : = build_replicated_int_cst (word_type,
6976 1 : width, max & ~(max >> 1));
6977 1 : tree signs;
6978 1 : if (code == PLUS_EXPR || code == MINUS_EXPR)
6979 : {
6980 1 : signs = make_ssa_name (word_type);
6981 1 : new_stmt = gimple_build_assign (signs,
6982 : BIT_XOR_EXPR, wvop0, wvop1);
6983 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6984 1 : tree b_low = make_ssa_name (word_type);
6985 1 : new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
6986 : wvop1, low_bits);
6987 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6988 1 : tree a_low = make_ssa_name (word_type);
6989 1 : if (code == PLUS_EXPR)
6990 1 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
6991 : wvop0, low_bits);
6992 : else
6993 0 : new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
6994 : wvop0, high_bits);
6995 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6996 1 : if (code == MINUS_EXPR)
6997 : {
6998 0 : new_stmt = gimple_build_assign (NULL_TREE,
6999 : BIT_NOT_EXPR, signs);
7000 0 : signs = make_ssa_name (word_type);
7001 0 : gimple_assign_set_lhs (new_stmt, signs);
7002 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7003 : new_stmt, gsi);
7004 : }
7005 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7006 : signs, high_bits);
7007 1 : signs = make_ssa_name (word_type);
7008 1 : gimple_assign_set_lhs (new_stmt, signs);
7009 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7010 1 : result_low = make_ssa_name (word_type);
7011 1 : new_stmt = gimple_build_assign (result_low, code,
7012 : a_low, b_low);
7013 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7014 : }
7015 : else /* if (code == NEGATE_EXPR) */
7016 : {
7017 0 : tree a_low = make_ssa_name (word_type);
7018 0 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7019 : wvop0, low_bits);
7020 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7021 0 : signs = make_ssa_name (word_type);
7022 0 : new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
7023 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7024 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7025 : signs, high_bits);
7026 0 : signs = make_ssa_name (word_type);
7027 0 : gimple_assign_set_lhs (new_stmt, signs);
7028 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7029 0 : result_low = make_ssa_name (word_type);
7030 0 : new_stmt = gimple_build_assign (result_low,
7031 : MINUS_EXPR, high_bits, a_low);
7032 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7033 : }
7034 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
7035 : result_low, signs);
7036 1 : result_low = make_ssa_name (word_type);
7037 1 : gimple_assign_set_lhs (new_stmt, result_low);
7038 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7039 : }
7040 : else
7041 : {
7042 1 : new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
7043 1 : result_low = make_ssa_name (word_type);
7044 1 : gimple_assign_set_lhs (new_stmt, result_low);
7045 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7046 :
7047 : }
7048 2 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
7049 : build1 (VIEW_CONVERT_EXPR,
7050 : vectype, result_low));
7051 2 : new_temp = make_ssa_name (vectype);
7052 2 : gimple_assign_set_lhs (new_stmt, new_temp);
7053 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7054 : }
7055 139199 : else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
7056 : {
7057 16 : tree mask;
7058 16 : if (masked_loop_p)
7059 16 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7060 : vec_num, vectype, i);
7061 : else
7062 : /* Dummy mask. */
7063 0 : mask = build_minus_one_cst (truth_type_for (vectype));
7064 16 : auto_vec<tree> vops (6);
7065 16 : vops.quick_push (mask);
7066 16 : vops.quick_push (vop0);
7067 16 : if (vop1)
7068 16 : vops.quick_push (vop1);
7069 16 : if (vop2)
7070 0 : vops.quick_push (vop2);
7071 16 : if (reduc_idx >= 0)
7072 : {
7073 : /* Perform the operation on active elements only and take
7074 : inactive elements from the reduction chain input. */
7075 8 : gcc_assert (!vop2);
7076 8 : vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
7077 : }
7078 : else
7079 : {
7080 8 : auto else_value = targetm.preferred_else_value
7081 8 : (cond_fn, vectype, vops.length () - 1, &vops[1]);
7082 8 : vops.quick_push (else_value);
7083 : }
7084 16 : if (len_loop_p)
7085 : {
7086 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
7087 0 : vec_num, vectype, i, 1, true);
7088 0 : signed char biasval
7089 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
7090 0 : tree bias = build_int_cst (intQI_type_node, biasval);
7091 0 : vops.quick_push (len);
7092 0 : vops.quick_push (bias);
7093 : }
7094 16 : gcall *call
7095 16 : = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
7096 : : cond_len_fn,
7097 : vops);
7098 16 : new_temp = make_ssa_name (vec_dest, call);
7099 16 : gimple_call_set_lhs (call, new_temp);
7100 16 : gimple_call_set_nothrow (call, true);
7101 16 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
7102 16 : new_stmt = call;
7103 16 : }
7104 : else
7105 : {
7106 139183 : tree mask = NULL_TREE;
7107 : /* When combining two masks check if either of them is elsewhere
7108 : combined with a loop mask, if that's the case we can mark that the
7109 : new combined mask doesn't need to be combined with a loop mask. */
7110 139183 : if (masked_loop_p
7111 139183 : && code == BIT_AND_EXPR
7112 139183 : && VECTOR_BOOLEAN_TYPE_P (vectype))
7113 : {
7114 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
7115 : {
7116 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7117 : vec_num, vectype, i);
7118 :
7119 0 : vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7120 : vop0, gsi);
7121 : }
7122 :
7123 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
7124 : {
7125 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7126 : vec_num, vectype, i);
7127 :
7128 0 : vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7129 : vop1, gsi);
7130 : }
7131 : }
7132 :
7133 139183 : new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
7134 139183 : new_temp = make_ssa_name (vec_dest, new_stmt);
7135 139183 : gimple_assign_set_lhs (new_stmt, new_temp);
7136 139183 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7137 139183 : if (using_emulated_vectors_p)
7138 : suppress_warning (new_stmt, OPT_Wvector_operation_performance);
7139 :
7140 : /* Enter the combined value into the vector cond hash so we don't
7141 : AND it with a loop mask again. */
7142 139183 : if (mask)
7143 0 : loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
7144 : }
7145 :
7146 139201 : if (vec_cvt_dest)
7147 : {
7148 3052 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
7149 3052 : new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
7150 : new_temp);
7151 3052 : new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
7152 3052 : gimple_assign_set_lhs (new_stmt, new_temp);
7153 3052 : vect_finish_stmt_generation (vinfo, stmt_info,
7154 : new_stmt, gsi);
7155 : }
7156 :
7157 139201 : slp_node->push_vec_def (new_stmt);
7158 : }
7159 :
7160 114895 : vec_oprnds0.release ();
7161 114895 : vec_oprnds1.release ();
7162 114895 : vec_oprnds2.release ();
7163 :
7164 114895 : return true;
7165 : }
7166 :
7167 : /* A helper function to ensure data reference DR_INFO's base alignment. */
7168 :
7169 : static void
7170 1992368 : ensure_base_align (dr_vec_info *dr_info)
7171 : {
7172 : /* Alignment is only analyzed for the first element of a DR group,
7173 : use that to look at base alignment we need to enforce. */
7174 1992368 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
7175 1449069 : dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
7176 :
7177 1992368 : gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
7178 :
7179 1992368 : if (dr_info->base_misaligned)
7180 : {
7181 170808 : tree base_decl = dr_info->base_decl;
7182 :
7183 : // We should only be able to increase the alignment of a base object if
7184 : // we know what its new alignment should be at compile time.
7185 170808 : unsigned HOST_WIDE_INT align_base_to =
7186 170808 : DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
7187 :
7188 170808 : if (decl_in_symtab_p (base_decl))
7189 4800 : symtab_node::get (base_decl)->increase_alignment (align_base_to);
7190 166008 : else if (DECL_ALIGN (base_decl) < align_base_to)
7191 : {
7192 132866 : SET_DECL_ALIGN (base_decl, align_base_to);
7193 132866 : DECL_USER_ALIGN (base_decl) = 1;
7194 : }
7195 170808 : dr_info->base_misaligned = false;
7196 : }
7197 1992368 : }
7198 :
7199 :
7200 : /* Function get_group_alias_ptr_type.
7201 :
7202 : Return the alias type for the group starting at FIRST_STMT_INFO. */
7203 :
7204 : static tree
7205 1655129 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
7206 : {
7207 1655129 : struct data_reference *first_dr, *next_dr;
7208 :
7209 1655129 : first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
7210 1655129 : stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
7211 3921880 : while (next_stmt_info)
7212 : {
7213 2452182 : next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
7214 4904364 : if (get_alias_set (DR_REF (first_dr))
7215 2452182 : != get_alias_set (DR_REF (next_dr)))
7216 : {
7217 185431 : if (dump_enabled_p ())
7218 30 : dump_printf_loc (MSG_NOTE, vect_location,
7219 : "conflicting alias set types.\n");
7220 185431 : return ptr_type_node;
7221 : }
7222 2266751 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7223 : }
7224 1469698 : return reference_alias_ptr_type (DR_REF (first_dr));
7225 : }
7226 :
7227 :
7228 : /* Function scan_operand_equal_p.
7229 :
7230 : Helper function for check_scan_store. Compare two references
7231 : with .GOMP_SIMD_LANE bases. */
7232 :
7233 : static bool
7234 1284 : scan_operand_equal_p (tree ref1, tree ref2)
7235 : {
7236 1284 : tree ref[2] = { ref1, ref2 };
7237 1284 : poly_int64 bitsize[2], bitpos[2];
7238 : tree offset[2], base[2];
7239 3852 : for (int i = 0; i < 2; ++i)
7240 : {
7241 2568 : machine_mode mode;
7242 2568 : int unsignedp, reversep, volatilep = 0;
7243 2568 : base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
7244 : &offset[i], &mode, &unsignedp,
7245 : &reversep, &volatilep);
7246 2568 : if (reversep || volatilep || maybe_ne (bitpos[i], 0))
7247 0 : return false;
7248 2568 : if (TREE_CODE (base[i]) == MEM_REF
7249 42 : && offset[i] == NULL_TREE
7250 2610 : && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
7251 : {
7252 42 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
7253 42 : if (is_gimple_assign (def_stmt)
7254 42 : && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
7255 42 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
7256 84 : && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
7257 : {
7258 42 : if (maybe_ne (mem_ref_offset (base[i]), 0))
7259 : return false;
7260 42 : base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
7261 42 : offset[i] = gimple_assign_rhs2 (def_stmt);
7262 : }
7263 : }
7264 : }
7265 :
7266 1284 : if (!operand_equal_p (base[0], base[1], 0))
7267 : return false;
7268 934 : if (maybe_ne (bitsize[0], bitsize[1]))
7269 : return false;
7270 934 : if (offset[0] != offset[1])
7271 : {
7272 916 : if (!offset[0] || !offset[1])
7273 : return false;
7274 916 : if (!operand_equal_p (offset[0], offset[1], 0))
7275 : {
7276 : tree step[2];
7277 0 : for (int i = 0; i < 2; ++i)
7278 : {
7279 0 : step[i] = integer_one_node;
7280 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7281 : {
7282 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7283 0 : if (is_gimple_assign (def_stmt)
7284 0 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
7285 0 : && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
7286 : == INTEGER_CST))
7287 : {
7288 0 : step[i] = gimple_assign_rhs2 (def_stmt);
7289 0 : offset[i] = gimple_assign_rhs1 (def_stmt);
7290 : }
7291 : }
7292 0 : else if (TREE_CODE (offset[i]) == MULT_EXPR)
7293 : {
7294 0 : step[i] = TREE_OPERAND (offset[i], 1);
7295 0 : offset[i] = TREE_OPERAND (offset[i], 0);
7296 : }
7297 0 : tree rhs1 = NULL_TREE;
7298 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7299 : {
7300 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7301 0 : if (gimple_assign_cast_p (def_stmt))
7302 0 : rhs1 = gimple_assign_rhs1 (def_stmt);
7303 : }
7304 0 : else if (CONVERT_EXPR_P (offset[i]))
7305 0 : rhs1 = TREE_OPERAND (offset[i], 0);
7306 0 : if (rhs1
7307 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7308 0 : && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
7309 0 : && (TYPE_PRECISION (TREE_TYPE (offset[i]))
7310 0 : >= TYPE_PRECISION (TREE_TYPE (rhs1))))
7311 0 : offset[i] = rhs1;
7312 : }
7313 0 : if (!operand_equal_p (offset[0], offset[1], 0)
7314 0 : || !operand_equal_p (step[0], step[1], 0))
7315 0 : return false;
7316 : }
7317 : }
7318 : return true;
7319 : }
7320 :
7321 :
7322 : enum scan_store_kind {
7323 : /* Normal permutation. */
7324 : scan_store_kind_perm,
7325 :
7326 : /* Whole vector left shift permutation with zero init. */
7327 : scan_store_kind_lshift_zero,
7328 :
7329 : /* Whole vector left shift permutation and VEC_COND_EXPR. */
7330 : scan_store_kind_lshift_cond
7331 : };
7332 :
7333 : /* Function check_scan_store.
7334 :
7335 : Verify if we can perform the needed permutations or whole vector shifts.
7336 : Return -1 on failure, otherwise exact log2 of vectype's nunits.
7337 : USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7338 : to do at each step. */
7339 :
7340 : static int
7341 1024 : scan_store_can_perm_p (tree vectype, tree init,
7342 : vec<enum scan_store_kind> *use_whole_vector = NULL)
7343 : {
7344 1024 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7345 1024 : unsigned HOST_WIDE_INT nunits;
7346 1024 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7347 : return -1;
7348 1024 : int units_log2 = exact_log2 (nunits);
7349 1024 : if (units_log2 <= 0)
7350 : return -1;
7351 :
7352 : int i;
7353 : enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
7354 4784 : for (i = 0; i <= units_log2; ++i)
7355 : {
7356 3760 : unsigned HOST_WIDE_INT j, k;
7357 3760 : enum scan_store_kind kind = scan_store_kind_perm;
7358 3760 : vec_perm_builder sel (nunits, nunits, 1);
7359 3760 : sel.quick_grow (nunits);
7360 3760 : if (i == units_log2)
7361 : {
7362 9728 : for (j = 0; j < nunits; ++j)
7363 8704 : sel[j] = nunits - 1;
7364 : }
7365 : else
7366 : {
7367 10416 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7368 7680 : sel[j] = j;
7369 26416 : for (k = 0; j < nunits; ++j, ++k)
7370 23680 : sel[j] = nunits + k;
7371 : }
7372 6496 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7373 3760 : if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
7374 : {
7375 0 : if (i == units_log2)
7376 : return -1;
7377 :
7378 0 : if (whole_vector_shift_kind == scan_store_kind_perm)
7379 : {
7380 0 : if (!can_implement_p (vec_shl_optab, vec_mode))
7381 : return -1;
7382 0 : whole_vector_shift_kind = scan_store_kind_lshift_zero;
7383 : /* Whole vector shifts shift in zeros, so if init is all zero
7384 : constant, there is no need to do anything further. */
7385 0 : if ((TREE_CODE (init) != INTEGER_CST
7386 0 : && TREE_CODE (init) != REAL_CST)
7387 0 : || !initializer_zerop (init))
7388 : {
7389 0 : tree masktype = truth_type_for (vectype);
7390 0 : if (!expand_vec_cond_expr_p (vectype, masktype))
7391 : return -1;
7392 : whole_vector_shift_kind = scan_store_kind_lshift_cond;
7393 : }
7394 : }
7395 0 : kind = whole_vector_shift_kind;
7396 : }
7397 3760 : if (use_whole_vector)
7398 : {
7399 1880 : if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
7400 0 : use_whole_vector->safe_grow_cleared (i, true);
7401 5640 : if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
7402 0 : use_whole_vector->safe_push (kind);
7403 : }
7404 3760 : }
7405 :
7406 : return units_log2;
7407 : }
7408 :
7409 :
7410 : /* Function check_scan_store.
7411 :
7412 : Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7413 :
7414 : static bool
7415 1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
7416 : enum vect_def_type rhs_dt, slp_tree slp_node,
7417 : slp_tree mask_node,
7418 : vect_memory_access_type memory_access_type)
7419 : {
7420 1076 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7421 1076 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7422 1076 : tree ref_type;
7423 :
7424 1076 : gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
7425 1076 : if (SLP_TREE_LANES (slp_node) > 1
7426 1076 : || mask_node
7427 1076 : || memory_access_type != VMAT_CONTIGUOUS
7428 1076 : || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
7429 1076 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
7430 1076 : || loop_vinfo == NULL
7431 1076 : || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
7432 1076 : || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
7433 1076 : || STMT_VINFO_GROUPED_ACCESS (stmt_info)
7434 1076 : || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
7435 1076 : || !integer_zerop (DR_INIT (dr_info->dr))
7436 1076 : || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
7437 2152 : || !alias_sets_conflict_p (get_alias_set (vectype),
7438 1076 : get_alias_set (TREE_TYPE (ref_type))))
7439 : {
7440 0 : if (dump_enabled_p ())
7441 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7442 : "unsupported OpenMP scan store.\n");
7443 0 : return false;
7444 : }
7445 :
7446 : /* We need to pattern match code built by OpenMP lowering and simplified
7447 : by following optimizations into something we can handle.
7448 : #pragma omp simd reduction(inscan,+:r)
7449 : for (...)
7450 : {
7451 : r += something ();
7452 : #pragma omp scan inclusive (r)
7453 : use (r);
7454 : }
7455 : shall have body with:
7456 : // Initialization for input phase, store the reduction initializer:
7457 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7458 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7459 : D.2042[_21] = 0;
7460 : // Actual input phase:
7461 : ...
7462 : r.0_5 = D.2042[_20];
7463 : _6 = _4 + r.0_5;
7464 : D.2042[_20] = _6;
7465 : // Initialization for scan phase:
7466 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7467 : _26 = D.2043[_25];
7468 : _27 = D.2042[_25];
7469 : _28 = _26 + _27;
7470 : D.2043[_25] = _28;
7471 : D.2042[_25] = _28;
7472 : // Actual scan phase:
7473 : ...
7474 : r.1_8 = D.2042[_20];
7475 : ...
7476 : The "omp simd array" variable D.2042 holds the privatized copy used
7477 : inside of the loop and D.2043 is another one that holds copies of
7478 : the current original list item. The separate GOMP_SIMD_LANE ifn
7479 : kinds are there in order to allow optimizing the initializer store
7480 : and combiner sequence, e.g. if it is originally some C++ish user
7481 : defined reduction, but allow the vectorizer to pattern recognize it
7482 : and turn into the appropriate vectorized scan.
7483 :
7484 : For exclusive scan, this is slightly different:
7485 : #pragma omp simd reduction(inscan,+:r)
7486 : for (...)
7487 : {
7488 : use (r);
7489 : #pragma omp scan exclusive (r)
7490 : r += something ();
7491 : }
7492 : shall have body with:
7493 : // Initialization for input phase, store the reduction initializer:
7494 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7495 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7496 : D.2042[_21] = 0;
7497 : // Actual input phase:
7498 : ...
7499 : r.0_5 = D.2042[_20];
7500 : _6 = _4 + r.0_5;
7501 : D.2042[_20] = _6;
7502 : // Initialization for scan phase:
7503 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7504 : _26 = D.2043[_25];
7505 : D.2044[_25] = _26;
7506 : _27 = D.2042[_25];
7507 : _28 = _26 + _27;
7508 : D.2043[_25] = _28;
7509 : // Actual scan phase:
7510 : ...
7511 : r.1_8 = D.2044[_20];
7512 : ... */
7513 :
7514 1076 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
7515 : {
7516 : /* Match the D.2042[_21] = 0; store above. Just require that
7517 : it is a constant or external definition store. */
7518 564 : if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
7519 : {
7520 0 : fail_init:
7521 0 : if (dump_enabled_p ())
7522 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7523 : "unsupported OpenMP scan initializer store.\n");
7524 0 : return false;
7525 : }
7526 :
7527 564 : if (! loop_vinfo->scan_map)
7528 322 : loop_vinfo->scan_map = new hash_map<tree, tree>;
7529 564 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7530 564 : tree &cached = loop_vinfo->scan_map->get_or_insert (var);
7531 564 : if (cached)
7532 0 : goto fail_init;
7533 564 : cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
7534 :
7535 : /* These stores can be vectorized normally. */
7536 564 : return true;
7537 : }
7538 :
7539 512 : if (rhs_dt != vect_internal_def)
7540 : {
7541 0 : fail:
7542 0 : if (dump_enabled_p ())
7543 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7544 : "unsupported OpenMP scan combiner pattern.\n");
7545 0 : return false;
7546 : }
7547 :
7548 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7549 512 : tree rhs = gimple_assign_rhs1 (stmt);
7550 512 : if (TREE_CODE (rhs) != SSA_NAME)
7551 0 : goto fail;
7552 :
7553 512 : gimple *other_store_stmt = NULL;
7554 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7555 512 : bool inscan_var_store
7556 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7557 :
7558 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7559 : {
7560 252 : if (!inscan_var_store)
7561 : {
7562 126 : use_operand_p use_p;
7563 126 : imm_use_iterator iter;
7564 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7565 : {
7566 252 : gimple *use_stmt = USE_STMT (use_p);
7567 252 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7568 126 : continue;
7569 126 : if (gimple_bb (use_stmt) != gimple_bb (stmt)
7570 126 : || !is_gimple_assign (use_stmt)
7571 126 : || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
7572 126 : || other_store_stmt
7573 252 : || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
7574 0 : goto fail;
7575 126 : other_store_stmt = use_stmt;
7576 0 : }
7577 126 : if (other_store_stmt == NULL)
7578 0 : goto fail;
7579 126 : rhs = gimple_assign_lhs (other_store_stmt);
7580 126 : if (!single_imm_use (rhs, &use_p, &other_store_stmt))
7581 0 : goto fail;
7582 : }
7583 : }
7584 260 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
7585 : {
7586 260 : use_operand_p use_p;
7587 260 : imm_use_iterator iter;
7588 1040 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7589 : {
7590 520 : gimple *use_stmt = USE_STMT (use_p);
7591 520 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7592 260 : continue;
7593 260 : if (other_store_stmt)
7594 0 : goto fail;
7595 260 : other_store_stmt = use_stmt;
7596 260 : }
7597 : }
7598 : else
7599 0 : goto fail;
7600 :
7601 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7602 512 : if (gimple_bb (def_stmt) != gimple_bb (stmt)
7603 512 : || !is_gimple_assign (def_stmt)
7604 1024 : || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
7605 0 : goto fail;
7606 :
7607 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7608 : /* For pointer addition, we should use the normal plus for the vector
7609 : operation. */
7610 512 : switch (code)
7611 : {
7612 0 : case POINTER_PLUS_EXPR:
7613 0 : code = PLUS_EXPR;
7614 0 : break;
7615 0 : case MULT_HIGHPART_EXPR:
7616 0 : goto fail;
7617 : default:
7618 : break;
7619 : }
7620 512 : if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
7621 0 : goto fail;
7622 :
7623 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7624 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7625 512 : if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
7626 0 : goto fail;
7627 :
7628 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7629 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7630 512 : if (gimple_bb (load1_stmt) != gimple_bb (stmt)
7631 512 : || !gimple_assign_load_p (load1_stmt)
7632 512 : || gimple_bb (load2_stmt) != gimple_bb (stmt)
7633 1024 : || !gimple_assign_load_p (load2_stmt))
7634 0 : goto fail;
7635 :
7636 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7637 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7638 512 : if (load1_stmt_info == NULL
7639 512 : || load2_stmt_info == NULL
7640 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
7641 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
7642 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
7643 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7644 0 : goto fail;
7645 :
7646 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
7647 : {
7648 126 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7649 126 : if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
7650 126 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
7651 0 : goto fail;
7652 126 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7653 126 : tree lrhs;
7654 126 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7655 : lrhs = rhs1;
7656 : else
7657 16 : lrhs = rhs2;
7658 126 : use_operand_p use_p;
7659 126 : imm_use_iterator iter;
7660 504 : FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
7661 : {
7662 252 : gimple *use_stmt = USE_STMT (use_p);
7663 252 : if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
7664 126 : continue;
7665 126 : if (other_store_stmt)
7666 0 : goto fail;
7667 126 : other_store_stmt = use_stmt;
7668 126 : }
7669 : }
7670 :
7671 512 : if (other_store_stmt == NULL)
7672 0 : goto fail;
7673 512 : if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
7674 512 : || !gimple_store_p (other_store_stmt))
7675 0 : goto fail;
7676 :
7677 512 : stmt_vec_info other_store_stmt_info
7678 512 : = loop_vinfo->lookup_stmt (other_store_stmt);
7679 512 : if (other_store_stmt_info == NULL
7680 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
7681 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7682 0 : goto fail;
7683 :
7684 512 : gimple *stmt1 = stmt;
7685 512 : gimple *stmt2 = other_store_stmt;
7686 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7687 : std::swap (stmt1, stmt2);
7688 512 : if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
7689 : gimple_assign_rhs1 (load2_stmt)))
7690 : {
7691 162 : std::swap (rhs1, rhs2);
7692 162 : std::swap (load1_stmt, load2_stmt);
7693 162 : std::swap (load1_stmt_info, load2_stmt_info);
7694 : }
7695 512 : if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
7696 : gimple_assign_rhs1 (load1_stmt)))
7697 0 : goto fail;
7698 :
7699 512 : tree var3 = NULL_TREE;
7700 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
7701 512 : && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
7702 : gimple_assign_rhs1 (load2_stmt)))
7703 0 : goto fail;
7704 512 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7705 : {
7706 252 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7707 252 : if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
7708 252 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
7709 0 : goto fail;
7710 252 : var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7711 252 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
7712 252 : || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
7713 504 : || lookup_attribute ("omp simd inscan exclusive",
7714 252 : DECL_ATTRIBUTES (var3)))
7715 0 : goto fail;
7716 : }
7717 :
7718 512 : dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
7719 512 : if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
7720 512 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
7721 0 : goto fail;
7722 :
7723 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7724 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
7725 512 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
7726 512 : || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
7727 1024 : || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7728 512 : == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
7729 0 : goto fail;
7730 :
7731 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7732 256 : std::swap (var1, var2);
7733 :
7734 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7735 : {
7736 252 : if (!lookup_attribute ("omp simd inscan exclusive",
7737 252 : DECL_ATTRIBUTES (var1)))
7738 0 : goto fail;
7739 252 : var1 = var3;
7740 : }
7741 :
7742 512 : if (loop_vinfo->scan_map == NULL)
7743 0 : goto fail;
7744 512 : tree *init = loop_vinfo->scan_map->get (var1);
7745 512 : if (init == NULL)
7746 0 : goto fail;
7747 :
7748 : /* The IL is as expected, now check if we can actually vectorize it.
7749 : Inclusive scan:
7750 : _26 = D.2043[_25];
7751 : _27 = D.2042[_25];
7752 : _28 = _26 + _27;
7753 : D.2043[_25] = _28;
7754 : D.2042[_25] = _28;
7755 : should be vectorized as (where _40 is the vectorized rhs
7756 : from the D.2042[_21] = 0; store):
7757 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7758 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7759 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7760 : _33 = _31 + _32;
7761 : // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7762 : _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7763 : _35 = _33 + _34;
7764 : // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7765 : // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7766 : _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7767 : _37 = _35 + _36;
7768 : // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7769 : // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7770 : _38 = _30 + _37;
7771 : _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7772 : MEM <vector(8) int> [(int *)&D.2043] = _39;
7773 : MEM <vector(8) int> [(int *)&D.2042] = _38;
7774 : Exclusive scan:
7775 : _26 = D.2043[_25];
7776 : D.2044[_25] = _26;
7777 : _27 = D.2042[_25];
7778 : _28 = _26 + _27;
7779 : D.2043[_25] = _28;
7780 : should be vectorized as (where _40 is the vectorized rhs
7781 : from the D.2042[_21] = 0; store):
7782 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7783 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7784 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7785 : _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7786 : _34 = _32 + _33;
7787 : // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7788 : // _31[3]+_31[4], ... _31[5]+.._31[6] };
7789 : _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7790 : _36 = _34 + _35;
7791 : // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7792 : // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7793 : _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7794 : _38 = _36 + _37;
7795 : // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7796 : // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7797 : _39 = _30 + _38;
7798 : _50 = _31 + _39;
7799 : _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7800 : MEM <vector(8) int> [(int *)&D.2044] = _39;
7801 : MEM <vector(8) int> [(int *)&D.2042] = _51; */
7802 512 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7803 512 : optab optab = optab_for_tree_code (code, vectype, optab_default);
7804 512 : if (!optab || !can_implement_p (optab, vec_mode))
7805 0 : goto fail;
7806 :
7807 512 : int units_log2 = scan_store_can_perm_p (vectype, *init);
7808 512 : if (units_log2 == -1)
7809 0 : goto fail;
7810 :
7811 : return true;
7812 : }
7813 :
7814 :
7815 : /* Function vectorizable_scan_store.
7816 :
7817 : Helper of vectorizable_score, arguments like on vectorizable_store.
7818 : Handle only the transformation, checking is done in check_scan_store. */
7819 :
7820 : static bool
7821 512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
7822 : slp_tree slp_node, gimple_stmt_iterator *gsi)
7823 : {
7824 512 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7825 512 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7826 512 : tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
7827 512 : tree vectype = SLP_TREE_VECTYPE (slp_node);
7828 :
7829 512 : if (dump_enabled_p ())
7830 492 : dump_printf_loc (MSG_NOTE, vect_location,
7831 : "transform scan store.\n");
7832 :
7833 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7834 512 : tree rhs = gimple_assign_rhs1 (stmt);
7835 512 : gcc_assert (TREE_CODE (rhs) == SSA_NAME);
7836 :
7837 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7838 512 : bool inscan_var_store
7839 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7840 :
7841 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7842 : {
7843 126 : use_operand_p use_p;
7844 126 : imm_use_iterator iter;
7845 252 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7846 : {
7847 126 : gimple *use_stmt = USE_STMT (use_p);
7848 126 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7849 0 : continue;
7850 126 : rhs = gimple_assign_lhs (use_stmt);
7851 126 : break;
7852 126 : }
7853 : }
7854 :
7855 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7856 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7857 512 : if (code == POINTER_PLUS_EXPR)
7858 0 : code = PLUS_EXPR;
7859 512 : gcc_assert (TREE_CODE_LENGTH (code) == binary_op
7860 : && commutative_tree_code (code));
7861 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7862 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7863 512 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
7864 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7865 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7866 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7867 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7868 512 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7869 512 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7870 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7871 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7872 :
7873 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7874 : {
7875 436 : std::swap (rhs1, rhs2);
7876 436 : std::swap (var1, var2);
7877 436 : std::swap (load1_dr_info, load2_dr_info);
7878 : }
7879 :
7880 512 : tree *init = loop_vinfo->scan_map->get (var1);
7881 512 : gcc_assert (init);
7882 :
7883 512 : unsigned HOST_WIDE_INT nunits;
7884 512 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7885 : gcc_unreachable ();
7886 512 : auto_vec<enum scan_store_kind, 16> use_whole_vector;
7887 512 : int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
7888 512 : gcc_assert (units_log2 > 0);
7889 512 : auto_vec<tree, 16> perms;
7890 512 : perms.quick_grow (units_log2 + 1);
7891 512 : tree zero_vec = NULL_TREE, masktype = NULL_TREE;
7892 2392 : for (int i = 0; i <= units_log2; ++i)
7893 : {
7894 1880 : unsigned HOST_WIDE_INT j, k;
7895 1880 : vec_perm_builder sel (nunits, nunits, 1);
7896 1880 : sel.quick_grow (nunits);
7897 1880 : if (i == units_log2)
7898 4864 : for (j = 0; j < nunits; ++j)
7899 4352 : sel[j] = nunits - 1;
7900 : else
7901 : {
7902 5208 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7903 3840 : sel[j] = j;
7904 13208 : for (k = 0; j < nunits; ++j, ++k)
7905 11840 : sel[j] = nunits + k;
7906 : }
7907 3248 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7908 1880 : if (!use_whole_vector.is_empty ()
7909 0 : && use_whole_vector[i] != scan_store_kind_perm)
7910 : {
7911 0 : if (zero_vec == NULL_TREE)
7912 0 : zero_vec = build_zero_cst (vectype);
7913 0 : if (masktype == NULL_TREE
7914 0 : && use_whole_vector[i] == scan_store_kind_lshift_cond)
7915 0 : masktype = truth_type_for (vectype);
7916 0 : perms[i] = vect_gen_perm_mask_any (vectype, indices);
7917 : }
7918 : else
7919 1880 : perms[i] = vect_gen_perm_mask_checked (vectype, indices);
7920 1880 : }
7921 :
7922 512 : tree vec_oprnd1 = NULL_TREE;
7923 512 : tree vec_oprnd2 = NULL_TREE;
7924 512 : tree vec_oprnd3 = NULL_TREE;
7925 512 : tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
7926 512 : tree dataref_offset = build_int_cst (ref_type, 0);
7927 512 : tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
7928 512 : tree ldataref_ptr = NULL_TREE;
7929 512 : tree orig = NULL_TREE;
7930 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7931 126 : ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
7932 : /* The initialization is invariant. */
7933 512 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
7934 512 : auto_vec<tree> vec_oprnds2;
7935 512 : auto_vec<tree> vec_oprnds3;
7936 512 : if (ldataref_ptr == NULL)
7937 : {
7938 : /* We want to lookup the vector operands of the reduction, not those
7939 : of the store - for SLP we have to use the proper SLP node for the
7940 : lookup, which should be the single child of the scan store. */
7941 386 : vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
7942 : rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
7943 : /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
7944 : them in SLP child order. So we have to swap here with logic
7945 : similar to above. */
7946 386 : stmt_vec_info load
7947 386 : = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
7948 386 : (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
7949 386 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
7950 386 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7951 386 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
7952 820 : for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
7953 494 : std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
7954 : }
7955 : else
7956 126 : vect_get_vec_defs (vinfo, slp_node,
7957 : rhs2, &vec_oprnds3);
7958 1248 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
7959 : {
7960 736 : if (ldataref_ptr == NULL)
7961 554 : vec_oprnd2 = vec_oprnds2[j];
7962 736 : vec_oprnd3 = vec_oprnds3[j];
7963 736 : if (j == 0)
7964 : orig = vec_oprnd3;
7965 224 : else if (!inscan_var_store)
7966 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
7967 :
7968 736 : if (ldataref_ptr)
7969 : {
7970 182 : vec_oprnd2 = make_ssa_name (vectype);
7971 182 : tree data_ref = fold_build2 (MEM_REF, vectype,
7972 : unshare_expr (ldataref_ptr),
7973 : dataref_offset);
7974 182 : vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
7975 182 : gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
7976 182 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
7977 : }
7978 :
7979 736 : tree v = vec_oprnd2;
7980 3068 : for (int i = 0; i < units_log2; ++i)
7981 : {
7982 2332 : tree new_temp = make_ssa_name (vectype);
7983 2332 : gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
7984 : (zero_vec
7985 0 : && (use_whole_vector[i]
7986 0 : != scan_store_kind_perm))
7987 : ? zero_vec : vec_oprnd1, v,
7988 2332 : perms[i]);
7989 2332 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
7990 :
7991 2332 : if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
7992 : {
7993 : /* Whole vector shift shifted in zero bits, but if *init
7994 : is not initializer_zerop, we need to replace those elements
7995 : with elements from vec_oprnd1. */
7996 0 : tree_vector_builder vb (masktype, nunits, 1);
7997 0 : for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
7998 0 : vb.quick_push (k < (HOST_WIDE_INT_1U << i)
7999 : ? boolean_false_node : boolean_true_node);
8000 :
8001 0 : tree new_temp2 = make_ssa_name (vectype);
8002 0 : g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
8003 : new_temp, vec_oprnd1);
8004 0 : vect_finish_stmt_generation (vinfo, stmt_info,
8005 : g, gsi);
8006 0 : new_temp = new_temp2;
8007 0 : }
8008 :
8009 : /* For exclusive scan, perform the perms[i] permutation once
8010 : more. */
8011 2332 : if (i == 0
8012 1100 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
8013 728 : && v == vec_oprnd2)
8014 : {
8015 364 : v = new_temp;
8016 364 : --i;
8017 364 : continue;
8018 : }
8019 :
8020 1968 : tree new_temp2 = make_ssa_name (vectype);
8021 1968 : g = gimple_build_assign (new_temp2, code, v, new_temp);
8022 1968 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8023 :
8024 1968 : v = new_temp2;
8025 : }
8026 :
8027 736 : tree new_temp = make_ssa_name (vectype);
8028 736 : gimple *g = gimple_build_assign (new_temp, code, orig, v);
8029 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8030 :
8031 736 : tree last_perm_arg = new_temp;
8032 : /* For exclusive scan, new_temp computed above is the exclusive scan
8033 : prefix sum. Turn it into inclusive prefix sum for the broadcast
8034 : of the last element into orig. */
8035 736 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
8036 : {
8037 364 : last_perm_arg = make_ssa_name (vectype);
8038 364 : g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
8039 364 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8040 : }
8041 :
8042 736 : orig = make_ssa_name (vectype);
8043 2208 : g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
8044 736 : last_perm_arg, perms[units_log2]);
8045 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8046 :
8047 736 : if (!inscan_var_store)
8048 : {
8049 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8050 : unshare_expr (dataref_ptr),
8051 : dataref_offset);
8052 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8053 368 : g = gimple_build_assign (data_ref, new_temp);
8054 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8055 : }
8056 : }
8057 :
8058 512 : if (inscan_var_store)
8059 624 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8060 : {
8061 368 : if (j != 0)
8062 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8063 :
8064 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8065 : unshare_expr (dataref_ptr),
8066 : dataref_offset);
8067 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8068 368 : gimple *g = gimple_build_assign (data_ref, orig);
8069 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8070 : }
8071 512 : return true;
8072 512 : }
8073 :
8074 :
8075 : /* Function vectorizable_store.
8076 :
8077 : Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8078 : that can be vectorized.
8079 : If COST_VEC is passed, calculate costs but don't change anything,
8080 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
8081 : it, and insert it at GSI.
8082 : Return true if STMT_INFO is vectorizable in this way. */
8083 :
8084 : static bool
8085 2118251 : vectorizable_store (vec_info *vinfo,
8086 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8087 : slp_tree slp_node,
8088 : stmt_vector_for_cost *cost_vec)
8089 : {
8090 2118251 : tree data_ref;
8091 2118251 : tree vec_oprnd = NULL_TREE;
8092 2118251 : tree elem_type;
8093 2118251 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
8094 2118251 : class loop *loop = NULL;
8095 2118251 : machine_mode vec_mode;
8096 2118251 : tree dummy;
8097 2118251 : enum vect_def_type rhs_dt = vect_unknown_def_type;
8098 2118251 : enum vect_def_type mask_dt = vect_unknown_def_type;
8099 2118251 : tree dataref_ptr = NULL_TREE;
8100 2118251 : tree dataref_offset = NULL_TREE;
8101 2118251 : int j;
8102 2118251 : stmt_vec_info first_stmt_info;
8103 2118251 : bool grouped_store;
8104 2118251 : unsigned int group_size, i;
8105 2118251 : unsigned int vec_num;
8106 2118251 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
8107 2118251 : tree aggr_type;
8108 2118251 : poly_uint64 vf;
8109 2118251 : vec_load_store_type vls_type;
8110 2118251 : tree ref_type;
8111 :
8112 2118251 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8113 : return false;
8114 :
8115 2118251 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8116 237977 : && cost_vec)
8117 : return false;
8118 :
8119 : /* Is vectorizable store? */
8120 :
8121 1880274 : tree mask_vectype = NULL_TREE;
8122 1880274 : slp_tree mask_node = NULL;
8123 1880274 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
8124 : {
8125 1807690 : tree scalar_dest = gimple_assign_lhs (assign);
8126 1807690 : if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
8127 1807690 : && is_pattern_stmt_p (stmt_info))
8128 1672 : scalar_dest = TREE_OPERAND (scalar_dest, 0);
8129 1807690 : if (TREE_CODE (scalar_dest) != ARRAY_REF
8130 1807690 : && TREE_CODE (scalar_dest) != BIT_FIELD_REF
8131 : && TREE_CODE (scalar_dest) != INDIRECT_REF
8132 : && TREE_CODE (scalar_dest) != COMPONENT_REF
8133 : && TREE_CODE (scalar_dest) != IMAGPART_EXPR
8134 : && TREE_CODE (scalar_dest) != REALPART_EXPR
8135 : && TREE_CODE (scalar_dest) != MEM_REF)
8136 : return false;
8137 : }
8138 : else
8139 : {
8140 752171 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
8141 13230 : if (!call || !gimple_call_internal_p (call))
8142 : return false;
8143 :
8144 8326 : internal_fn ifn = gimple_call_internal_fn (call);
8145 8326 : if (!internal_store_fn_p (ifn))
8146 : return false;
8147 :
8148 1881 : int mask_index = internal_fn_mask_index (ifn);
8149 1881 : if (mask_index >= 0)
8150 1881 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
8151 1881 : if (mask_index >= 0
8152 1881 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
8153 : &mask_node, &mask_dt,
8154 : &mask_vectype))
8155 : return false;
8156 : }
8157 :
8158 1379342 : tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
8159 1379342 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8160 :
8161 1379342 : if (loop_vinfo)
8162 : {
8163 227418 : loop = LOOP_VINFO_LOOP (loop_vinfo);
8164 227418 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8165 : }
8166 : else
8167 : vf = 1;
8168 1379342 : vec_num = vect_get_num_copies (vinfo, slp_node);
8169 :
8170 : /* FORNOW. This restriction should be relaxed. */
8171 1379342 : if (loop
8172 1379617 : && nested_in_vect_loop_p (loop, stmt_info)
8173 1379625 : && vec_num > 1)
8174 : {
8175 8 : if (dump_enabled_p ())
8176 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8177 : "multiple types in nested loop.\n");
8178 8 : return false;
8179 : }
8180 :
8181 1379334 : slp_tree op_node;
8182 1379334 : if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
8183 : &op_node, &rhs_dt, &rhs_vectype, &vls_type))
8184 : return false;
8185 :
8186 1379310 : elem_type = TREE_TYPE (vectype);
8187 1379310 : vec_mode = TYPE_MODE (vectype);
8188 :
8189 1379310 : if (!STMT_VINFO_DATA_REF (stmt_info))
8190 : return false;
8191 :
8192 1379310 : vect_load_store_data _ls_data{};
8193 1379310 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
8194 1379310 : if (cost_vec
8195 1379310 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
8196 : vls_type, &_ls_data))
8197 : return false;
8198 : /* Temporary aliases to analysis data, should not be modified through
8199 : these. */
8200 1378694 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
8201 1378694 : const dr_alignment_support alignment_support_scheme
8202 : = ls.alignment_support_scheme;
8203 1378694 : const int misalignment = ls.misalignment;
8204 1378694 : const poly_int64 poffset = ls.poffset;
8205 :
8206 1378694 : if (slp_node->ldst_lanes
8207 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
8208 : {
8209 0 : if (dump_enabled_p ())
8210 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8211 : "discovered store-lane but cannot use it.\n");
8212 0 : return false;
8213 : }
8214 :
8215 1378694 : if (mask_node)
8216 : {
8217 1791 : if (memory_access_type == VMAT_CONTIGUOUS)
8218 : {
8219 616 : if (!VECTOR_MODE_P (vec_mode)
8220 3066 : || !can_vec_mask_load_store_p (vec_mode,
8221 1533 : TYPE_MODE (mask_vectype), false))
8222 114 : return false;
8223 : }
8224 258 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
8225 258 : && (!mat_gather_scatter_p (memory_access_type)
8226 242 : || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
8227 170 : && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
8228 : {
8229 16 : if (dump_enabled_p ())
8230 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8231 : "unsupported access type for masked store.\n");
8232 16 : return false;
8233 : }
8234 242 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
8235 : {
8236 72 : if (dump_enabled_p ())
8237 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8238 : "unsupported masked emulated scatter.\n");
8239 72 : return false;
8240 : }
8241 : }
8242 : else
8243 : {
8244 : /* FORNOW. In some cases can vectorize even if data-type not supported
8245 : (e.g. - array initialization with 0). */
8246 1376903 : if (!can_implement_p (mov_optab, vec_mode))
8247 : return false;
8248 : }
8249 :
8250 1378492 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
8251 1378492 : grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
8252 2550342 : && !mat_gather_scatter_p (memory_access_type));
8253 1171850 : if (grouped_store)
8254 : {
8255 1171850 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8256 1171850 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8257 1171850 : group_size = DR_GROUP_SIZE (first_stmt_info);
8258 : }
8259 : else
8260 : {
8261 1378492 : first_stmt_info = stmt_info;
8262 1378492 : first_dr_info = dr_info;
8263 : group_size = 1;
8264 : }
8265 :
8266 1378492 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
8267 : {
8268 1076 : if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
8269 : mask_node, memory_access_type))
8270 : return false;
8271 : }
8272 :
8273 2756216 : bool costing_p = cost_vec;
8274 1377724 : if (costing_p) /* transformation not required. */
8275 : {
8276 825672 : if (loop_vinfo
8277 163348 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
8278 76788 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
8279 : vls_type, group_size, &ls,
8280 : mask_node);
8281 :
8282 825672 : if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
8283 825672 : || (mask_node
8284 1050 : && !vect_maybe_update_slp_op_vectype (mask_node,
8285 : mask_vectype)))
8286 : {
8287 0 : if (dump_enabled_p ())
8288 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8289 : "incompatible vector types for invariants\n");
8290 0 : return false;
8291 : }
8292 :
8293 825672 : if (dump_enabled_p ()
8294 : && memory_access_type != VMAT_ELEMENTWISE
8295 15091 : && memory_access_type != VMAT_STRIDED_SLP
8296 14435 : && memory_access_type != VMAT_INVARIANT
8297 840107 : && alignment_support_scheme != dr_aligned)
8298 4995 : dump_printf_loc (MSG_NOTE, vect_location,
8299 : "Vectorizing an unaligned access.\n");
8300 : }
8301 :
8302 : /* Transform. */
8303 :
8304 1378492 : ensure_base_align (dr_info);
8305 :
8306 1378492 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
8307 : {
8308 1024 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
8309 1024 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8310 1024 : if (costing_p)
8311 : {
8312 512 : unsigned int inside_cost = 0, prologue_cost = 0;
8313 512 : if (vls_type == VLS_STORE_INVARIANT)
8314 0 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8315 : slp_node, 0, vect_prologue);
8316 512 : vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
8317 : alignment_support_scheme, misalignment,
8318 : &inside_cost, cost_vec);
8319 :
8320 512 : if (dump_enabled_p ())
8321 492 : dump_printf_loc (MSG_NOTE, vect_location,
8322 : "vect_model_store_cost: inside_cost = %d, "
8323 : "prologue_cost = %d .\n",
8324 : inside_cost, prologue_cost);
8325 :
8326 512 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8327 512 : slp_node->data = new vect_load_store_data (std::move (ls));
8328 :
8329 512 : return true;
8330 : }
8331 512 : return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
8332 : }
8333 :
8334 : /* FORNOW */
8335 1377468 : gcc_assert (!grouped_store
8336 : || !loop
8337 : || !nested_in_vect_loop_p (loop, stmt_info));
8338 :
8339 1377468 : grouped_store = false;
8340 1377468 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
8341 1377468 : gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
8342 : || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
8343 1377468 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8344 :
8345 1377468 : ref_type = get_group_alias_ptr_type (first_stmt_info);
8346 :
8347 1377468 : if (!costing_p && dump_enabled_p ())
8348 12257 : dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
8349 :
8350 1377468 : if (memory_access_type == VMAT_ELEMENTWISE
8351 1377468 : || memory_access_type == VMAT_STRIDED_SLP)
8352 : {
8353 29190 : unsigned inside_cost = 0, prologue_cost = 0;
8354 29190 : gimple_stmt_iterator incr_gsi;
8355 29190 : bool insert_after;
8356 29190 : tree offvar = NULL_TREE;
8357 29190 : tree ivstep;
8358 29190 : tree running_off;
8359 29190 : tree stride_base, stride_step, alias_off;
8360 29190 : tree vec_oprnd = NULL_TREE;
8361 29190 : tree dr_offset;
8362 : /* Checked by get_load_store_type. */
8363 29190 : unsigned int const_nunits = nunits.to_constant ();
8364 :
8365 29190 : gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
8366 29190 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
8367 :
8368 29190 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
8369 29190 : stride_base
8370 29190 : = fold_build_pointer_plus
8371 : (DR_BASE_ADDRESS (first_dr_info->dr),
8372 : size_binop (PLUS_EXPR,
8373 : convert_to_ptrofftype (dr_offset),
8374 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
8375 29190 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
8376 :
8377 : /* For a store with loop-invariant (but other than power-of-2)
8378 : stride (i.e. not a grouped access) like so:
8379 :
8380 : for (i = 0; i < n; i += stride)
8381 : array[i] = ...;
8382 :
8383 : we generate a new induction variable and new stores from
8384 : the components of the (vectorized) rhs:
8385 :
8386 : for (j = 0; ; j += VF*stride)
8387 : vectemp = ...;
8388 : tmp1 = vectemp[0];
8389 : array[j] = tmp1;
8390 : tmp2 = vectemp[1];
8391 : array[j + stride] = tmp2;
8392 : ...
8393 : */
8394 :
8395 : /* ??? Modify local copies of alignment_support_scheme and
8396 : misalignment, but this part of analysis should be done
8397 : earlier and remembered, likewise the chosen load mode. */
8398 29190 : const dr_alignment_support tem = alignment_support_scheme;
8399 29190 : dr_alignment_support alignment_support_scheme = tem;
8400 29190 : const int tem2 = misalignment;
8401 29190 : int misalignment = tem2;
8402 :
8403 29190 : unsigned nstores = const_nunits;
8404 29190 : unsigned lnel = 1;
8405 29190 : tree ltype = elem_type;
8406 29190 : tree lvectype = vectype;
8407 29190 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
8408 29190 : if (n == const_nunits)
8409 : {
8410 2939 : int mis_align = dr_misalignment (first_dr_info, vectype);
8411 : /* With VF > 1 we advance the DR by step, if that is constant
8412 : and only aligned when performed VF times, DR alignment
8413 : analysis can analyze this as aligned since it assumes
8414 : contiguous accesses. But that is not how we code generate
8415 : here, so adjust for this. */
8416 2939 : if (maybe_gt (vf, 1u)
8417 4464 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8418 4235 : DR_TARGET_ALIGNMENT (first_dr_info)))
8419 229 : mis_align = -1;
8420 2939 : dr_alignment_support dr_align
8421 2939 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
8422 : mis_align);
8423 2939 : if (dr_align == dr_aligned
8424 2939 : || dr_align == dr_unaligned_supported)
8425 : {
8426 29190 : nstores = 1;
8427 29190 : lnel = const_nunits;
8428 29190 : ltype = vectype;
8429 29190 : lvectype = vectype;
8430 29190 : alignment_support_scheme = dr_align;
8431 29190 : misalignment = mis_align;
8432 : }
8433 : }
8434 26251 : else if (n > 1)
8435 : {
8436 1967 : nstores = const_nunits / n;
8437 1967 : lnel = n;
8438 1967 : ltype = build_vector_type (elem_type, n);
8439 1967 : lvectype = vectype;
8440 1967 : int mis_align = dr_misalignment (first_dr_info, ltype);
8441 1967 : if (maybe_gt (vf, 1u)
8442 3934 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8443 3292 : DR_TARGET_ALIGNMENT (first_dr_info)))
8444 642 : mis_align = -1;
8445 1967 : dr_alignment_support dr_align
8446 1967 : = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
8447 : mis_align);
8448 1967 : alignment_support_scheme = dr_align;
8449 1967 : misalignment = mis_align;
8450 :
8451 : /* First check if vec_extract optab doesn't support extraction
8452 : of vector elts directly. */
8453 1967 : scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
8454 1967 : machine_mode vmode;
8455 3934 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8456 2139 : || !related_vector_mode (TYPE_MODE (vectype), elmode,
8457 1967 : n).exists (&vmode)
8458 1773 : || (convert_optab_handler (vec_extract_optab,
8459 1773 : TYPE_MODE (vectype), vmode)
8460 : == CODE_FOR_nothing)
8461 1967 : || !(dr_align == dr_aligned
8462 172 : || dr_align == dr_unaligned_supported))
8463 : {
8464 : /* Try to avoid emitting an extract of vector elements
8465 : by performing the extracts using an integer type of the
8466 : same size, extracting from a vector of those and then
8467 : re-interpreting it as the original vector type if
8468 : supported. */
8469 1795 : unsigned lsize = n * GET_MODE_BITSIZE (elmode);
8470 1795 : unsigned int lnunits = const_nunits / n;
8471 : /* If we can't construct such a vector fall back to
8472 : element extracts from the original vector type and
8473 : element size stores. */
8474 1795 : if (int_mode_for_size (lsize, 0).exists (&elmode)
8475 1795 : && VECTOR_MODE_P (TYPE_MODE (vectype))
8476 1795 : && related_vector_mode (TYPE_MODE (vectype), elmode,
8477 1795 : lnunits).exists (&vmode)
8478 1767 : && (convert_optab_handler (vec_extract_optab,
8479 : vmode, elmode)
8480 : != CODE_FOR_nothing))
8481 : {
8482 1767 : nstores = lnunits;
8483 1767 : lnel = n;
8484 1767 : ltype = build_nonstandard_integer_type (lsize, 1);
8485 1767 : lvectype = build_vector_type (ltype, nstores);
8486 : }
8487 : /* Else fall back to vector extraction anyway.
8488 : Fewer stores are more important than avoiding spilling
8489 : of the vector we extract from. Compared to the
8490 : construction case in vectorizable_load no store-forwarding
8491 : issue exists here for reasonable archs. But only
8492 : if the store is supported. */
8493 28 : else if (!(dr_align == dr_aligned
8494 28 : || dr_align == dr_unaligned_supported))
8495 : {
8496 29190 : nstores = const_nunits;
8497 29190 : lnel = 1;
8498 29190 : ltype = elem_type;
8499 29190 : lvectype = vectype;
8500 : }
8501 : }
8502 : }
8503 :
8504 29190 : if (costing_p)
8505 : {
8506 : /* Record the decomposition type for target access during costing. */
8507 25812 : ls.ls_type = lvectype;
8508 25812 : ls.ls_eltype = ltype;
8509 : }
8510 : else
8511 3378 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
8512 :
8513 29190 : unsigned align;
8514 29190 : if (alignment_support_scheme == dr_aligned)
8515 1241 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
8516 : else
8517 27949 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
8518 : /* Alignment is at most the access size if we do multiple stores. */
8519 29190 : if (nstores > 1)
8520 26251 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
8521 29190 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
8522 29190 : int ncopies = vec_num;
8523 :
8524 29190 : if (!costing_p)
8525 : {
8526 3378 : ivstep = stride_step;
8527 :
8528 3378 : tree increment = fold_convert (TREE_TYPE (ivstep),
8529 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
8530 :
8531 3378 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
8532 : increment);
8533 :
8534 3378 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
8535 :
8536 3378 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
8537 3378 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
8538 3378 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
8539 : else
8540 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
8541 : true, NULL_TREE, true,
8542 : GSI_SAME_STMT);
8543 :
8544 3378 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
8545 : insert_after, &offvar, NULL,
8546 3378 : LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
8547 :
8548 3378 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
8549 : }
8550 :
8551 29190 : alias_off = build_int_cst (ref_type, 0);
8552 29190 : auto_vec<tree> vec_oprnds;
8553 : /* For costing some adjacent vector stores, we'd like to cost with
8554 : the total number of them once instead of cost each one by one. */
8555 29190 : unsigned int n_adjacent_stores = 0;
8556 29190 : running_off = offvar;
8557 29190 : if (!costing_p)
8558 3378 : vect_get_slp_defs (op_node, &vec_oprnds);
8559 29190 : unsigned int group_el = 0;
8560 29190 : unsigned HOST_WIDE_INT elsz
8561 29190 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
8562 69603 : for (j = 0; j < ncopies; j++)
8563 : {
8564 40413 : if (!costing_p)
8565 : {
8566 5219 : vec_oprnd = vec_oprnds[j];
8567 : /* Pun the vector to extract from if necessary. */
8568 5219 : if (lvectype != vectype)
8569 : {
8570 1008 : tree tem = make_ssa_name (lvectype);
8571 1008 : tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
8572 1008 : gimple *pun = gimple_build_assign (tem, cvt);
8573 1008 : vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
8574 1008 : vec_oprnd = tem;
8575 : }
8576 : }
8577 179653 : for (i = 0; i < nstores; i++)
8578 : {
8579 139240 : if (costing_p)
8580 : {
8581 123331 : n_adjacent_stores++;
8582 123331 : continue;
8583 : }
8584 15909 : tree newref, newoff;
8585 15909 : gimple *incr, *assign;
8586 15909 : tree size = TYPE_SIZE (ltype);
8587 : /* Extract the i'th component. */
8588 15909 : tree pos = fold_build2 (MULT_EXPR, bitsizetype,
8589 : bitsize_int (i), size);
8590 15909 : tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
8591 : size, pos);
8592 :
8593 15909 : elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
8594 : GSI_SAME_STMT);
8595 :
8596 15909 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
8597 15909 : group_el * elsz);
8598 15909 : newref = build2 (MEM_REF, ltype, running_off, this_off);
8599 15909 : vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
8600 :
8601 : /* And store it to *running_off. */
8602 15909 : assign = gimple_build_assign (newref, elem);
8603 15909 : vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
8604 :
8605 15909 : group_el += lnel;
8606 15909 : if (group_el == group_size)
8607 : {
8608 14272 : newoff = copy_ssa_name (running_off, NULL);
8609 14272 : incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
8610 : running_off, stride_step);
8611 14272 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
8612 :
8613 14272 : running_off = newoff;
8614 14272 : group_el = 0;
8615 : }
8616 : }
8617 : }
8618 :
8619 29190 : if (costing_p)
8620 : {
8621 25812 : if (n_adjacent_stores > 0)
8622 : {
8623 : /* Take a single lane vector type store as scalar
8624 : store to avoid ICE like 110776. */
8625 25812 : if (VECTOR_TYPE_P (ltype)
8626 25812 : && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
8627 1612 : vect_get_store_cost (vinfo, stmt_info, slp_node,
8628 : n_adjacent_stores, alignment_support_scheme,
8629 : misalignment, &inside_cost, cost_vec);
8630 : else
8631 24200 : inside_cost
8632 24200 : += record_stmt_cost (cost_vec, n_adjacent_stores,
8633 : scalar_store, slp_node, 0, vect_body);
8634 : /* Only need vector deconstruction when there is more
8635 : than one store. */
8636 25812 : if (nstores > 1)
8637 23784 : inside_cost
8638 23784 : += record_stmt_cost (cost_vec, ncopies,
8639 : vec_deconstruct, slp_node, 0, vect_body);
8640 : }
8641 25812 : if (dump_enabled_p ())
8642 656 : dump_printf_loc (MSG_NOTE, vect_location,
8643 : "vect_model_store_cost: inside_cost = %d, "
8644 : "prologue_cost = %d .\n",
8645 : inside_cost, prologue_cost);
8646 :
8647 25812 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8648 25812 : slp_node->data = new vect_load_store_data (std::move (ls));
8649 : }
8650 :
8651 29190 : return true;
8652 29190 : }
8653 :
8654 1348278 : gcc_assert (alignment_support_scheme);
8655 1348278 : vec_loop_masks *loop_masks
8656 196354 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
8657 1348278 : ? &LOOP_VINFO_MASKS (loop_vinfo)
8658 11 : : NULL);
8659 11 : vec_loop_lens *loop_lens
8660 196354 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
8661 : ? &LOOP_VINFO_LENS (loop_vinfo)
8662 0 : : NULL);
8663 :
8664 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8665 : are some difference here. We cannot enable both the lens and masks
8666 : during transform but it is allowed during analysis.
8667 : Shouldn't go with length-based approach if fully masked. */
8668 1348278 : if (cost_vec == NULL)
8669 : /* The cost_vec is NULL during transform. */
8670 548930 : gcc_assert ((!loop_lens || !loop_masks));
8671 :
8672 : /* Targets with store-lane instructions must not require explicit
8673 : realignment. vect_supportable_dr_alignment always returns either
8674 : dr_aligned or dr_unaligned_supported for masked operations. */
8675 1348278 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
8676 : && !mask_node
8677 : && !loop_masks)
8678 : || alignment_support_scheme == dr_aligned
8679 : || alignment_support_scheme == dr_unaligned_supported);
8680 :
8681 1348278 : tree offset = NULL_TREE;
8682 1348278 : if (!known_eq (poffset, 0))
8683 4647 : offset = size_int (poffset);
8684 :
8685 1348278 : tree dr_increment;
8686 1348278 : tree dr_bump;
8687 :
8688 1348278 : tree vec_offset = NULL_TREE;
8689 1348278 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8690 : {
8691 1456 : aggr_type = NULL_TREE;
8692 1456 : dr_increment = NULL_TREE;
8693 1456 : dr_bump = NULL_TREE;
8694 : }
8695 1346822 : else if (mat_gather_scatter_p (memory_access_type))
8696 : {
8697 0 : aggr_type = elem_type;
8698 0 : if (!costing_p)
8699 : {
8700 0 : tree vtype = ls.ls_type ? ls.ls_type : vectype;
8701 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
8702 : ls.strided_offset_vectype,
8703 : loop_vinfo, gsi,
8704 : &dr_increment, &dr_bump,
8705 : &vec_offset);
8706 : }
8707 : }
8708 : else
8709 : {
8710 1346822 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8711 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
8712 : else
8713 : aggr_type = vectype;
8714 1346822 : if (!costing_p)
8715 : {
8716 548459 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
8717 : memory_access_type);
8718 548459 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
8719 : memory_access_type);
8720 : }
8721 : }
8722 :
8723 1348278 : if (loop_vinfo && mask_node && !costing_p)
8724 539 : LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
8725 :
8726 : /* In case the vectorization factor (VF) is bigger than the number
8727 : of elements that we can fit in a vectype (nunits), we have to generate
8728 : more than one vector stmt - i.e - we need to "unroll" the
8729 : vector stmt by a factor VF/nunits. */
8730 :
8731 1348278 : auto_vec<tree> dr_chain (group_size);
8732 1348278 : auto_vec<tree> vec_masks;
8733 1348278 : tree vec_mask = NULL;
8734 1348278 : auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
8735 6077212 : for (i = 0; i < group_size; i++)
8736 3380656 : gvec_oprnds.quick_push (new auto_vec<tree> ());
8737 :
8738 1348278 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8739 : {
8740 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
8741 :
8742 0 : if (costing_p)
8743 : /* Update all incoming store operand nodes, the general handling
8744 : above only handles the mask and the first store operand node. */
8745 0 : for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
8746 0 : if (child != mask_node
8747 0 : && !vect_maybe_update_slp_op_vectype (child, vectype))
8748 : {
8749 0 : if (dump_enabled_p ())
8750 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8751 : "incompatible vector types for invariants\n");
8752 0 : return false;
8753 : }
8754 0 : unsigned inside_cost = 0, prologue_cost = 0;
8755 : /* For costing some adjacent vector stores, we'd like to cost with
8756 : the total number of them once instead of cost each one by one. */
8757 0 : unsigned int n_adjacent_stores = 0;
8758 0 : int ncopies = vec_num / group_size;
8759 0 : for (j = 0; j < ncopies; j++)
8760 : {
8761 0 : if (j == 0)
8762 : {
8763 0 : if (!costing_p)
8764 : {
8765 0 : if (mask_node)
8766 : {
8767 0 : vect_get_slp_defs (mask_node, &vec_masks);
8768 0 : vec_mask = vec_masks[0];
8769 : }
8770 0 : dataref_ptr
8771 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8772 : aggr_type, NULL, offset, &dummy,
8773 : gsi, NULL, false, dr_increment);
8774 : }
8775 : }
8776 0 : else if (!costing_p)
8777 : {
8778 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8779 0 : if (mask_node)
8780 0 : vec_mask = vec_masks[j];
8781 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
8782 : stmt_info, dr_bump);
8783 : }
8784 :
8785 0 : if (costing_p)
8786 : {
8787 0 : n_adjacent_stores += group_size;
8788 0 : continue;
8789 : }
8790 :
8791 : /* Get an array into which we can store the individual vectors. */
8792 0 : tree vec_array = create_vector_array (vectype, group_size);
8793 :
8794 : /* Invalidate the current contents of VEC_ARRAY. This should
8795 : become an RTL clobber too, which prevents the vector registers
8796 : from being upward-exposed. */
8797 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8798 :
8799 : /* Store the individual vectors into the array. */
8800 0 : for (i = 0; i < group_size; i++)
8801 : {
8802 0 : slp_tree child;
8803 0 : if (i == 0 || !mask_node)
8804 0 : child = SLP_TREE_CHILDREN (slp_node)[i];
8805 : else
8806 0 : child = SLP_TREE_CHILDREN (slp_node)[i + 1];
8807 0 : vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
8808 0 : write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
8809 : i);
8810 : }
8811 :
8812 0 : tree final_mask = NULL;
8813 0 : tree final_len = NULL;
8814 0 : tree bias = NULL;
8815 0 : if (loop_masks)
8816 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
8817 : ncopies, vectype, j);
8818 0 : if (vec_mask)
8819 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
8820 : vec_mask, gsi);
8821 :
8822 0 : if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
8823 : {
8824 0 : if (loop_lens)
8825 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
8826 : ncopies, vectype, j, 1, true);
8827 : else
8828 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8829 0 : signed char biasval
8830 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
8831 0 : bias = build_int_cst (intQI_type_node, biasval);
8832 0 : if (!final_mask)
8833 : {
8834 0 : mask_vectype = truth_type_for (vectype);
8835 0 : final_mask = build_minus_one_cst (mask_vectype);
8836 : }
8837 : }
8838 :
8839 0 : gcall *call;
8840 0 : if (final_len && final_mask)
8841 : {
8842 : /* Emit:
8843 : MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8844 : LEN, BIAS, VEC_ARRAY). */
8845 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8846 0 : tree alias_ptr = build_int_cst (ref_type, align);
8847 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
8848 : dataref_ptr, alias_ptr,
8849 : final_mask, final_len, bias,
8850 : vec_array);
8851 : }
8852 0 : else if (final_mask)
8853 : {
8854 : /* Emit:
8855 : MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8856 : VEC_ARRAY). */
8857 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8858 0 : tree alias_ptr = build_int_cst (ref_type, align);
8859 0 : call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
8860 : dataref_ptr, alias_ptr,
8861 : final_mask, vec_array);
8862 : }
8863 : else
8864 : {
8865 : /* Emit:
8866 : MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8867 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8868 0 : call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
8869 0 : gimple_call_set_lhs (call, data_ref);
8870 : }
8871 0 : gimple_call_set_nothrow (call, true);
8872 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
8873 :
8874 : /* Record that VEC_ARRAY is now dead. */
8875 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8876 : }
8877 :
8878 0 : if (costing_p)
8879 : {
8880 0 : if (n_adjacent_stores > 0)
8881 0 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
8882 : alignment_support_scheme, misalignment,
8883 : &inside_cost, cost_vec);
8884 0 : if (dump_enabled_p ())
8885 0 : dump_printf_loc (MSG_NOTE, vect_location,
8886 : "vect_model_store_cost: inside_cost = %d, "
8887 : "prologue_cost = %d .\n",
8888 : inside_cost, prologue_cost);
8889 :
8890 0 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8891 0 : slp_node->data = new vect_load_store_data (std::move (ls));
8892 : }
8893 :
8894 0 : return true;
8895 : }
8896 :
8897 1348278 : if (mat_gather_scatter_p (memory_access_type))
8898 : {
8899 1456 : gcc_assert (!grouped_store || ls.ls_type);
8900 1456 : if (ls.ls_type)
8901 0 : vectype = ls.ls_type;
8902 1456 : auto_vec<tree> vec_offsets;
8903 1456 : unsigned int inside_cost = 0, prologue_cost = 0;
8904 1456 : int num_stmts = vec_num;
8905 3311 : for (j = 0; j < num_stmts; j++)
8906 : {
8907 1855 : gimple *new_stmt;
8908 1855 : if (j == 0)
8909 : {
8910 1456 : if (costing_p && vls_type == VLS_STORE_INVARIANT)
8911 210 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8912 : slp_node, 0, vect_prologue);
8913 : else if (!costing_p)
8914 : {
8915 : /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8916 : DR_CHAIN is of size 1. */
8917 471 : gcc_assert (group_size == 1);
8918 471 : vect_get_slp_defs (op_node, gvec_oprnds[0]);
8919 471 : if (mask_node)
8920 70 : vect_get_slp_defs (mask_node, &vec_masks);
8921 :
8922 471 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8923 471 : vect_get_gather_scatter_ops (loop, slp_node,
8924 : &dataref_ptr, &vec_offsets);
8925 : else
8926 0 : dataref_ptr
8927 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8928 : aggr_type, NULL, offset,
8929 : &dummy, gsi, NULL, false,
8930 : dr_increment);
8931 : }
8932 : }
8933 399 : else if (!costing_p)
8934 : {
8935 35 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8936 35 : if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8937 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
8938 : gsi, stmt_info, dr_bump);
8939 : }
8940 :
8941 2571 : new_stmt = NULL;
8942 716 : if (!costing_p)
8943 : {
8944 506 : vec_oprnd = (*gvec_oprnds[0])[j];
8945 506 : if (mask_node)
8946 90 : vec_mask = vec_masks[j];
8947 : /* We should have caught mismatched types earlier. */
8948 506 : gcc_assert (ls.ls_type
8949 : || useless_type_conversion_p
8950 : (vectype, TREE_TYPE (vec_oprnd)));
8951 : }
8952 506 : tree final_mask = NULL_TREE;
8953 2361 : tree final_len = NULL_TREE;
8954 2361 : tree bias = NULL_TREE;
8955 506 : if (!costing_p)
8956 : {
8957 506 : if (loop_masks)
8958 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi,
8959 : loop_masks, num_stmts,
8960 : vectype, j);
8961 506 : if (vec_mask)
8962 90 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
8963 : final_mask, vec_mask, gsi);
8964 : }
8965 :
8966 1855 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
8967 1855 : tree alias_align_ptr = build_int_cst (ref_type, align);
8968 1855 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
8969 : {
8970 0 : if (costing_p)
8971 : {
8972 0 : if (ls.supported_offset_vectype
8973 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
8974 : vec_offset))
8975 0 : inside_cost
8976 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
8977 : slp_node, 0, vect_body);
8978 0 : if (ls.supported_scale)
8979 0 : inside_cost
8980 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
8981 : slp_node, 0, vect_body);
8982 :
8983 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
8984 0 : inside_cost
8985 0 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
8986 : slp_node, 0, vect_body);
8987 1855 : continue;
8988 0 : }
8989 :
8990 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8991 0 : vec_offset = vec_offsets[j];
8992 :
8993 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
8994 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
8995 :
8996 : /* Perform the offset conversion and scaling if necessary. */
8997 0 : if (!strided
8998 0 : && (ls.supported_offset_vectype || ls.supported_scale))
8999 : {
9000 0 : gimple_seq stmts = NULL;
9001 0 : if (ls.supported_offset_vectype)
9002 0 : vec_offset = gimple_convert
9003 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
9004 0 : if (ls.supported_scale)
9005 : {
9006 : /* Only scale the vec_offset if we haven't already. */
9007 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
9008 0 : || j == 0)
9009 : {
9010 0 : tree mult_cst = build_int_cst
9011 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
9012 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
9013 0 : tree mult = build_vector_from_val
9014 0 : (TREE_TYPE (vec_offset), mult_cst);
9015 0 : vec_offset = gimple_build
9016 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
9017 : vec_offset, mult);
9018 : }
9019 0 : scale = size_int (ls.supported_scale);
9020 : }
9021 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9022 : }
9023 :
9024 0 : if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
9025 : {
9026 0 : if (loop_lens)
9027 0 : final_len = vect_get_loop_len (loop_vinfo, gsi,
9028 : loop_lens, num_stmts,
9029 : vectype, j, 1, true);
9030 : else
9031 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9032 :
9033 0 : signed char biasval
9034 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9035 0 : bias = build_int_cst (intQI_type_node, biasval);
9036 0 : if (!final_mask)
9037 : {
9038 0 : mask_vectype = truth_type_for (vectype);
9039 0 : final_mask = build_minus_one_cst (mask_vectype);
9040 : }
9041 : }
9042 :
9043 0 : if (ls.ls_type)
9044 : {
9045 0 : gimple *conv_stmt
9046 0 : = gimple_build_assign (make_ssa_name (vectype),
9047 : VIEW_CONVERT_EXPR,
9048 : build1 (VIEW_CONVERT_EXPR, vectype,
9049 : vec_oprnd));
9050 0 : vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
9051 : gsi);
9052 0 : vec_oprnd = gimple_get_lhs (conv_stmt);
9053 : }
9054 :
9055 0 : gcall *call;
9056 0 : if (final_len && final_mask)
9057 : {
9058 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
9059 0 : call = gimple_build_call_internal (
9060 : IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
9061 : alias_align_ptr,
9062 : vec_offset, scale, vec_oprnd, final_mask, final_len,
9063 : bias);
9064 : else
9065 : /* Non-vector offset indicates that prefer to take
9066 : MASK_LEN_STRIDED_STORE instead of the
9067 : IFN_MASK_SCATTER_STORE with direct stride arg.
9068 : Similar to the gather case we have checked the
9069 : alignment for a scatter already and assume
9070 : that the strided store has the same requirements. */
9071 0 : call = gimple_build_call_internal (
9072 : IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
9073 : vec_offset, vec_oprnd, final_mask, final_len, bias);
9074 : }
9075 0 : else if (final_mask)
9076 0 : call = gimple_build_call_internal
9077 0 : (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
9078 : alias_align_ptr,
9079 : vec_offset, scale, vec_oprnd, final_mask);
9080 : else
9081 0 : call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
9082 : dataref_ptr,
9083 : alias_align_ptr,
9084 : vec_offset,
9085 : scale, vec_oprnd);
9086 0 : gimple_call_set_nothrow (call, true);
9087 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9088 0 : new_stmt = call;
9089 : }
9090 1855 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
9091 : {
9092 : /* The builtin decls path for scatter is legacy, x86 only. */
9093 330 : gcc_assert (nunits.is_constant ()
9094 : && (!final_mask
9095 : || SCALAR_INT_MODE_P
9096 : (TYPE_MODE (TREE_TYPE (final_mask)))));
9097 330 : if (costing_p)
9098 : {
9099 199 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9100 199 : inside_cost
9101 199 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9102 : slp_node, 0, vect_body);
9103 199 : continue;
9104 199 : }
9105 :
9106 131 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9107 131 : poly_uint64 offset_nunits
9108 131 : = TYPE_VECTOR_SUBPARTS (offset_vectype);
9109 131 : if (known_eq (nunits, offset_nunits))
9110 : {
9111 55 : new_stmt = vect_build_one_scatter_store_call
9112 110 : (vinfo, stmt_info, slp_node, gsi,
9113 55 : ls.gs.decl, dataref_ptr, vec_offsets[j],
9114 : vec_oprnd, final_mask);
9115 55 : vect_finish_stmt_generation (vinfo, stmt_info,
9116 : new_stmt, gsi);
9117 : }
9118 76 : else if (known_eq (nunits, offset_nunits * 2))
9119 : {
9120 : /* We have a offset vector with half the number of
9121 : lanes but the builtins will store full vectype
9122 : data from the lower lanes. */
9123 30 : new_stmt = vect_build_one_scatter_store_call
9124 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9125 30 : dataref_ptr, vec_offsets[2 * j],
9126 : vec_oprnd, final_mask);
9127 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9128 : new_stmt, gsi);
9129 30 : int count = nunits.to_constant ();
9130 30 : vec_perm_builder sel (count, count, 1);
9131 30 : sel.quick_grow (count);
9132 382 : for (int i = 0; i < count; ++i)
9133 352 : sel[i] = i | (count / 2);
9134 30 : vec_perm_indices indices (sel, 2, count);
9135 30 : tree perm_mask
9136 30 : = vect_gen_perm_mask_checked (vectype, indices);
9137 30 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
9138 : vec_oprnd, vec_oprnd,
9139 : perm_mask);
9140 30 : vec_oprnd = make_ssa_name (vectype);
9141 30 : gimple_set_lhs (new_stmt, vec_oprnd);
9142 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9143 : new_stmt, gsi);
9144 30 : if (final_mask)
9145 : {
9146 20 : new_stmt = gimple_build_assign (NULL_TREE,
9147 : VEC_UNPACK_HI_EXPR,
9148 : final_mask);
9149 20 : final_mask = make_ssa_name
9150 20 : (truth_type_for (offset_vectype));
9151 20 : gimple_set_lhs (new_stmt, final_mask);
9152 20 : vect_finish_stmt_generation (vinfo, stmt_info,
9153 : new_stmt, gsi);
9154 : }
9155 :
9156 30 : new_stmt = vect_build_one_scatter_store_call
9157 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9158 30 : dataref_ptr, vec_offsets[2 * j + 1],
9159 : vec_oprnd, final_mask);
9160 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9161 : new_stmt, gsi);
9162 30 : }
9163 46 : else if (known_eq (nunits * 2, offset_nunits))
9164 : {
9165 : /* We have a offset vector with double the number of
9166 : lanes. Select the low/high part accordingly. */
9167 46 : vec_offset = vec_offsets[j / 2];
9168 46 : if (j & 1)
9169 : {
9170 23 : int count = offset_nunits.to_constant ();
9171 23 : vec_perm_builder sel (count, count, 1);
9172 23 : sel.quick_grow (count);
9173 263 : for (int i = 0; i < count; ++i)
9174 240 : sel[i] = i | (count / 2);
9175 23 : vec_perm_indices indices (sel, 2, count);
9176 23 : tree perm_mask = vect_gen_perm_mask_checked
9177 23 : (TREE_TYPE (vec_offset), indices);
9178 23 : new_stmt = gimple_build_assign (NULL_TREE,
9179 : VEC_PERM_EXPR,
9180 : vec_offset,
9181 : vec_offset,
9182 : perm_mask);
9183 23 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
9184 23 : gimple_set_lhs (new_stmt, vec_offset);
9185 23 : vect_finish_stmt_generation (vinfo, stmt_info,
9186 : new_stmt, gsi);
9187 23 : }
9188 :
9189 46 : new_stmt = vect_build_one_scatter_store_call
9190 46 : (vinfo, stmt_info, slp_node, gsi,
9191 : ls.gs.decl, dataref_ptr, vec_offset,
9192 : vec_oprnd, final_mask);
9193 46 : vect_finish_stmt_generation (vinfo, stmt_info,
9194 : new_stmt, gsi);
9195 : }
9196 : else
9197 0 : gcc_unreachable ();
9198 : }
9199 : else
9200 : {
9201 : /* Emulated scatter. */
9202 1525 : gcc_assert (!final_mask);
9203 1525 : if (costing_p)
9204 : {
9205 1150 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9206 : /* For emulated scatter N offset vector element extracts
9207 : (we assume the scalar scaling and ptr + offset add is
9208 : consumed by the load). */
9209 1150 : inside_cost
9210 1150 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9211 : slp_node, 0, vect_body);
9212 : /* N scalar stores plus extracting the elements. */
9213 1150 : inside_cost
9214 1150 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9215 : slp_node, 0, vect_body);
9216 1150 : inside_cost
9217 1150 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9218 : slp_node, 0, vect_body);
9219 1150 : continue;
9220 1150 : }
9221 :
9222 375 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9223 375 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
9224 375 : unsigned HOST_WIDE_INT const_offset_nunits
9225 375 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
9226 375 : vec<constructor_elt, va_gc> *ctor_elts;
9227 375 : vec_alloc (ctor_elts, const_nunits);
9228 375 : gimple_seq stmts = NULL;
9229 375 : tree elt_type = TREE_TYPE (vectype);
9230 375 : unsigned HOST_WIDE_INT elt_size
9231 375 : = tree_to_uhwi (TYPE_SIZE (elt_type));
9232 : /* We support offset vectors with more elements
9233 : than the data vector for now. */
9234 375 : unsigned HOST_WIDE_INT factor
9235 : = const_offset_nunits / const_nunits;
9236 375 : vec_offset = vec_offsets[j / factor];
9237 375 : unsigned elt_offset
9238 375 : = (j % factor) * const_nunits;
9239 375 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
9240 375 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9241 375 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
9242 1523 : for (unsigned k = 0; k < const_nunits; ++k)
9243 : {
9244 : /* Compute the offsetted pointer. */
9245 1148 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
9246 : bitsize_int (k + elt_offset));
9247 1148 : tree idx
9248 2296 : = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
9249 1148 : vec_offset, TYPE_SIZE (idx_type), boff);
9250 1148 : idx = gimple_convert (&stmts, sizetype, idx);
9251 1148 : idx = gimple_build (&stmts, MULT_EXPR, sizetype,
9252 : idx, scale);
9253 1148 : tree ptr
9254 1148 : = gimple_build (&stmts, PLUS_EXPR,
9255 1148 : TREE_TYPE (dataref_ptr),
9256 : dataref_ptr, idx);
9257 1148 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
9258 : /* Extract the element to be stored. */
9259 1148 : tree elt
9260 2296 : = gimple_build (&stmts, BIT_FIELD_REF,
9261 1148 : TREE_TYPE (vectype),
9262 1148 : vec_oprnd, TYPE_SIZE (elt_type),
9263 1148 : bitsize_int (k * elt_size));
9264 1148 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9265 1148 : stmts = NULL;
9266 1148 : tree ref
9267 1148 : = build2 (MEM_REF, ltype, ptr,
9268 : build_int_cst (ref_type, 0));
9269 1148 : new_stmt = gimple_build_assign (ref, elt);
9270 1148 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9271 : }
9272 :
9273 375 : slp_node->push_vec_def (new_stmt);
9274 : }
9275 : }
9276 :
9277 1456 : if (costing_p)
9278 : {
9279 985 : if (dump_enabled_p ())
9280 78 : dump_printf_loc (MSG_NOTE, vect_location,
9281 : "vect_model_store_cost: inside_cost = %d, "
9282 : "prologue_cost = %d .\n",
9283 : inside_cost, prologue_cost);
9284 985 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9285 985 : slp_node->data = new vect_load_store_data (std::move (ls));
9286 : }
9287 :
9288 1456 : return true;
9289 1456 : }
9290 :
9291 1346822 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
9292 : || memory_access_type == VMAT_CONTIGUOUS_DOWN
9293 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
9294 :
9295 1346822 : unsigned inside_cost = 0, prologue_cost = 0;
9296 : /* For costing some adjacent vector stores, we'd like to cost with
9297 : the total number of them once instead of cost each one by one. */
9298 1346822 : unsigned int n_adjacent_stores = 0;
9299 1346822 : auto_vec<tree> result_chain (group_size);
9300 1346822 : auto_vec<tree, 1> vec_oprnds;
9301 1346822 : gimple *new_stmt;
9302 1346822 : if (!costing_p)
9303 : {
9304 : /* Get vectorized arguments for SLP_NODE. */
9305 548459 : vect_get_slp_defs (op_node, &vec_oprnds);
9306 548459 : vec_oprnd = vec_oprnds[0];
9307 548459 : if (mask_node)
9308 : {
9309 469 : vect_get_slp_defs (mask_node, &vec_masks);
9310 469 : vec_mask = vec_masks[0];
9311 : }
9312 : }
9313 :
9314 : /* We should have caught mismatched types earlier. */
9315 548459 : gcc_assert (costing_p
9316 : || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
9317 1346822 : bool simd_lane_access_p
9318 1346822 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
9319 1346822 : if (!costing_p
9320 1346822 : && simd_lane_access_p
9321 4362 : && !loop_masks
9322 4362 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
9323 4362 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
9324 4362 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
9325 4362 : && integer_zerop (DR_INIT (first_dr_info->dr))
9326 1351184 : && alias_sets_conflict_p (get_alias_set (aggr_type),
9327 4362 : get_alias_set (TREE_TYPE (ref_type))))
9328 : {
9329 4354 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
9330 4354 : dataref_offset = build_int_cst (ref_type, 0);
9331 : }
9332 1342468 : else if (!costing_p)
9333 1088202 : dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
9334 : simd_lane_access_p ? loop : NULL,
9335 : offset, &dummy, gsi, NULL,
9336 : simd_lane_access_p, dr_increment);
9337 :
9338 1346822 : new_stmt = NULL;
9339 1346822 : gcc_assert (!grouped_store);
9340 2994163 : for (i = 0; i < vec_num; i++)
9341 : {
9342 1647341 : if (!costing_p)
9343 679414 : vec_oprnd = vec_oprnds[i];
9344 :
9345 1647341 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
9346 : {
9347 3327 : if (costing_p)
9348 2189 : inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
9349 : slp_node, 0, vect_body);
9350 : else
9351 : {
9352 1138 : tree perm_mask = perm_mask_for_reverse (vectype);
9353 1138 : tree new_temp = make_ssa_name (vectype);
9354 :
9355 : /* Generate the permute statement. */
9356 1138 : gimple *perm_stmt
9357 1138 : = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
9358 : vec_oprnd, perm_mask);
9359 1138 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9360 :
9361 1138 : perm_stmt = SSA_NAME_DEF_STMT (new_temp);
9362 1647341 : vec_oprnd = new_temp;
9363 : }
9364 : }
9365 :
9366 1647341 : if (costing_p)
9367 : {
9368 967927 : n_adjacent_stores++;
9369 967927 : continue;
9370 : }
9371 :
9372 679414 : tree final_mask = NULL_TREE;
9373 679414 : tree final_len = NULL_TREE;
9374 679414 : tree bias = NULL_TREE;
9375 679414 : if (loop_masks)
9376 77 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
9377 : vec_num, vectype, i);
9378 679414 : if (vec_mask)
9379 670 : vec_mask = vec_masks[i];
9380 670 : if (vec_mask)
9381 670 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
9382 : vec_mask, gsi);
9383 :
9384 679414 : if (i > 0)
9385 : /* Bump the vector pointer. */
9386 130955 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
9387 : dr_bump);
9388 :
9389 679414 : unsigned misalign;
9390 679414 : unsigned HOST_WIDE_INT align;
9391 679414 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
9392 679414 : if (alignment_support_scheme == dr_aligned)
9393 : misalign = 0;
9394 310996 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
9395 : {
9396 162276 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
9397 162276 : misalign = 0;
9398 : }
9399 : else
9400 148720 : misalign = misalignment;
9401 679414 : if (dataref_offset == NULL_TREE
9402 674051 : && TREE_CODE (dataref_ptr) == SSA_NAME)
9403 184385 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, misalign);
9404 679414 : align = least_bit_hwi (misalign | align);
9405 :
9406 : /* Compute IFN when LOOP_LENS or final_mask valid. */
9407 679414 : machine_mode vmode = TYPE_MODE (vectype);
9408 679414 : machine_mode new_vmode = vmode;
9409 679414 : internal_fn partial_ifn = IFN_LAST;
9410 679414 : if (loop_lens)
9411 : {
9412 0 : opt_machine_mode new_ovmode
9413 0 : = get_len_load_store_mode (vmode, false, &partial_ifn);
9414 0 : new_vmode = new_ovmode.require ();
9415 0 : unsigned factor
9416 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
9417 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
9418 : vec_num, vectype, i, factor, true);
9419 : }
9420 679414 : else if (final_mask)
9421 : {
9422 682 : if (!can_vec_mask_load_store_p (vmode,
9423 682 : TYPE_MODE (TREE_TYPE (final_mask)),
9424 : false, &partial_ifn))
9425 0 : gcc_unreachable ();
9426 : }
9427 :
9428 679414 : if (partial_ifn == IFN_MASK_LEN_STORE)
9429 : {
9430 0 : if (!final_len)
9431 : {
9432 : /* Pass VF value to 'len' argument of
9433 : MASK_LEN_STORE if LOOP_LENS is invalid. */
9434 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9435 : }
9436 0 : if (!final_mask)
9437 : {
9438 : /* Pass all ones value to 'mask' argument of
9439 : MASK_LEN_STORE if final_mask is invalid. */
9440 0 : mask_vectype = truth_type_for (vectype);
9441 0 : final_mask = build_minus_one_cst (mask_vectype);
9442 : }
9443 : }
9444 679414 : if (final_len)
9445 : {
9446 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9447 0 : bias = build_int_cst (intQI_type_node, biasval);
9448 : }
9449 :
9450 : /* Arguments are ready. Create the new vector stmt. */
9451 679414 : if (final_len)
9452 : {
9453 0 : gcall *call;
9454 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9455 : /* Need conversion if it's wrapped with VnQI. */
9456 0 : if (vmode != new_vmode)
9457 : {
9458 0 : tree new_vtype
9459 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
9460 : new_vmode);
9461 0 : tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
9462 0 : vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
9463 0 : gassign *new_stmt
9464 0 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
9465 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9466 0 : vec_oprnd = var;
9467 : }
9468 :
9469 0 : if (partial_ifn == IFN_MASK_LEN_STORE)
9470 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
9471 : dataref_ptr, ptr, final_mask,
9472 : final_len, bias, vec_oprnd);
9473 : else
9474 0 : call = gimple_build_call_internal (IFN_LEN_STORE, 5,
9475 : dataref_ptr, ptr, final_len,
9476 : bias, vec_oprnd);
9477 0 : gimple_call_set_nothrow (call, true);
9478 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9479 0 : new_stmt = call;
9480 : }
9481 679414 : else if (final_mask)
9482 : {
9483 682 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9484 682 : gcall *call
9485 682 : = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
9486 : ptr, final_mask, vec_oprnd);
9487 682 : gimple_call_set_nothrow (call, true);
9488 682 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9489 682 : new_stmt = call;
9490 : }
9491 : else
9492 : {
9493 678732 : data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
9494 : dataref_offset ? dataref_offset
9495 : : build_int_cst (ref_type, 0));
9496 678732 : if (alignment_support_scheme == dr_aligned
9497 678732 : && align >= TYPE_ALIGN_UNIT (vectype))
9498 : ;
9499 : else
9500 310484 : TREE_TYPE (data_ref)
9501 620968 : = build_aligned_type (TREE_TYPE (data_ref),
9502 : align * BITS_PER_UNIT);
9503 678732 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
9504 678732 : new_stmt = gimple_build_assign (data_ref, vec_oprnd);
9505 678732 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9506 : }
9507 : }
9508 :
9509 1346822 : if (costing_p)
9510 : {
9511 798363 : if (n_adjacent_stores > 0)
9512 798363 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
9513 : alignment_support_scheme, misalignment,
9514 : &inside_cost, cost_vec);
9515 :
9516 : /* When vectorizing a store into the function result assign
9517 : a penalty if the function returns in a multi-register location.
9518 : In this case we assume we'll end up with having to spill the
9519 : vector result and do piecewise loads as a conservative estimate. */
9520 798363 : tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
9521 798363 : if (base
9522 798363 : && (TREE_CODE (base) == RESULT_DECL
9523 747988 : || (DECL_P (base) && cfun_returns (base)))
9524 860554 : && !aggregate_value_p (base, cfun->decl))
9525 : {
9526 11121 : rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
9527 : /* ??? Handle PARALLEL in some way. */
9528 11121 : if (REG_P (reg))
9529 : {
9530 10915 : int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
9531 : /* Assume that a single reg-reg move is possible and cheap,
9532 : do not account for vector to gp register move cost. */
9533 10915 : if (nregs > 1)
9534 : {
9535 : /* Spill. */
9536 10087 : prologue_cost
9537 10087 : += record_stmt_cost (cost_vec, 1, vector_store,
9538 : slp_node, 0, vect_epilogue);
9539 : /* Loads. */
9540 10087 : prologue_cost
9541 10087 : += record_stmt_cost (cost_vec, nregs, scalar_load,
9542 : slp_node, 0, vect_epilogue);
9543 : }
9544 : }
9545 : }
9546 798363 : if (dump_enabled_p ())
9547 13865 : dump_printf_loc (MSG_NOTE, vect_location,
9548 : "vect_model_store_cost: inside_cost = %d, "
9549 : "prologue_cost = %d .\n",
9550 : inside_cost, prologue_cost);
9551 :
9552 798363 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9553 798363 : slp_node->data = new vect_load_store_data (std::move (ls));
9554 : }
9555 :
9556 1346822 : return true;
9557 2726132 : }
9558 :
9559 : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9560 : VECTOR_CST mask. No checks are made that the target platform supports the
9561 : mask, so callers may wish to test can_vec_perm_const_p separately, or use
9562 : vect_gen_perm_mask_checked. */
9563 :
9564 : tree
9565 61967 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
9566 : {
9567 61967 : tree mask_type;
9568 :
9569 61967 : poly_uint64 nunits = sel.length ();
9570 61967 : gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
9571 :
9572 61967 : mask_type = build_vector_type (ssizetype, nunits);
9573 61967 : return vec_perm_indices_to_tree (mask_type, sel);
9574 : }
9575 :
9576 : /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9577 : i.e. that the target supports the pattern _for arbitrary input vectors_. */
9578 :
9579 : tree
9580 59100 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
9581 : {
9582 59100 : machine_mode vmode = TYPE_MODE (vectype);
9583 59100 : gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
9584 59100 : return vect_gen_perm_mask_any (vectype, sel);
9585 : }
9586 :
9587 : /* Given a vector variable X and Y, that was generated for the scalar
9588 : STMT_INFO, generate instructions to permute the vector elements of X and Y
9589 : using permutation mask MASK_VEC, insert them at *GSI and return the
9590 : permuted vector variable. */
9591 :
9592 : static tree
9593 1445 : permute_vec_elements (vec_info *vinfo,
9594 : tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
9595 : gimple_stmt_iterator *gsi)
9596 : {
9597 1445 : tree vectype = TREE_TYPE (x);
9598 1445 : tree perm_dest, data_ref;
9599 1445 : gimple *perm_stmt;
9600 :
9601 1445 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
9602 1445 : if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
9603 1445 : perm_dest = vect_create_destination_var (scalar_dest, vectype);
9604 : else
9605 0 : perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
9606 1445 : data_ref = make_ssa_name (perm_dest);
9607 :
9608 : /* Generate the permute statement. */
9609 1445 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
9610 1445 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9611 :
9612 1445 : return data_ref;
9613 : }
9614 :
9615 : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9616 : inserting them on the loops preheader edge. Returns true if we
9617 : were successful in doing so (and thus STMT_INFO can be moved then),
9618 : otherwise returns false. HOIST_P indicates if we want to hoist the
9619 : definitions of all SSA uses, it would be false when we are costing. */
9620 :
9621 : static bool
9622 3994 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
9623 : {
9624 3994 : ssa_op_iter i;
9625 3994 : use_operand_p use_p;
9626 3994 : auto_vec<use_operand_p, 8> to_hoist;
9627 :
9628 7559 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
9629 : {
9630 3595 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9631 3595 : if (!gimple_nop_p (def_stmt)
9632 3595 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
9633 : {
9634 : /* Make sure we don't need to recurse. While we could do
9635 : so in simple cases when there are more complex use webs
9636 : we don't have an easy way to preserve stmt order to fulfil
9637 : dependencies within them. */
9638 93 : tree op2;
9639 93 : ssa_op_iter i2;
9640 93 : if (gimple_code (def_stmt) == GIMPLE_PHI
9641 93 : || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
9642 : == NULL_DEF_OPERAND_P))
9643 30 : return false;
9644 188 : FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
9645 : {
9646 125 : gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
9647 125 : if (!gimple_nop_p (def_stmt2)
9648 125 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
9649 : return false;
9650 : }
9651 63 : to_hoist.safe_push (use_p);
9652 : }
9653 : }
9654 :
9655 7928 : if (to_hoist.is_empty ())
9656 : return true;
9657 :
9658 39 : if (!hoist_p)
9659 : return true;
9660 :
9661 : /* Instead of moving defs we copy them so we can zero their UID to not
9662 : confuse dominance queries in the preheader. */
9663 5 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
9664 20 : for (use_operand_p use_p : to_hoist)
9665 : {
9666 5 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9667 5 : gimple *copy = gimple_copy (def_stmt);
9668 5 : gimple_set_uid (copy, 0);
9669 5 : def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
9670 5 : tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
9671 5 : update_stmt (copy);
9672 5 : def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
9673 5 : SET_DEF (def_p, new_def);
9674 5 : SET_USE (use_p, new_def);
9675 5 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
9676 : }
9677 :
9678 : return true;
9679 3994 : }
9680 :
9681 : /* vectorizable_load.
9682 :
9683 : Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9684 : that can be vectorized.
9685 : If COST_VEC is passed, calculate costs but don't change anything,
9686 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
9687 : it, and insert it at GSI.
9688 : Return true if STMT_INFO is vectorizable in this way. */
9689 :
9690 : static bool
9691 2179307 : vectorizable_load (vec_info *vinfo,
9692 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
9693 : slp_tree slp_node,
9694 : stmt_vector_for_cost *cost_vec)
9695 : {
9696 2179307 : tree scalar_dest;
9697 2179307 : tree vec_dest = NULL;
9698 2179307 : tree data_ref = NULL;
9699 2179307 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
9700 2179307 : class loop *loop = NULL;
9701 2179307 : class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
9702 2179307 : bool nested_in_vect_loop = false;
9703 2179307 : tree elem_type;
9704 : /* Avoid false positive uninitialized warning, see PR110652. */
9705 2179307 : tree new_temp = NULL_TREE;
9706 2179307 : machine_mode mode;
9707 2179307 : tree dummy;
9708 2179307 : tree dataref_ptr = NULL_TREE;
9709 2179307 : tree dataref_offset = NULL_TREE;
9710 2179307 : int i, j;
9711 2179307 : unsigned int group_size;
9712 2179307 : poly_uint64 group_gap_adj;
9713 2179307 : tree msq = NULL_TREE, lsq;
9714 2179307 : tree realignment_token = NULL_TREE;
9715 2179307 : gphi *phi = NULL;
9716 2179307 : bool grouped_load = false;
9717 2179307 : stmt_vec_info first_stmt_info;
9718 2179307 : stmt_vec_info first_stmt_info_for_drptr = NULL;
9719 2179307 : bool compute_in_loop = false;
9720 2179307 : class loop *at_loop;
9721 2179307 : int vec_num;
9722 2179307 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
9723 2179307 : poly_uint64 vf;
9724 2179307 : tree aggr_type;
9725 2179307 : tree ref_type;
9726 2179307 : enum vect_def_type mask_dt = vect_unknown_def_type;
9727 2179307 : enum vect_def_type els_dt = vect_unknown_def_type;
9728 :
9729 2179307 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
9730 : return false;
9731 :
9732 2179307 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
9733 237977 : && cost_vec)
9734 : return false;
9735 :
9736 1941330 : if (!STMT_VINFO_DATA_REF (stmt_info))
9737 : return false;
9738 :
9739 1552079 : tree mask_vectype = NULL_TREE;
9740 1552079 : tree els = NULL_TREE; tree els_vectype = NULL_TREE;
9741 :
9742 1552079 : int mask_index = -1;
9743 1552079 : int els_index = -1;
9744 1552079 : slp_tree mask_node = NULL;
9745 1552079 : slp_tree els_op = NULL;
9746 1552079 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
9747 : {
9748 1547646 : scalar_dest = gimple_assign_lhs (assign);
9749 1547646 : if (TREE_CODE (scalar_dest) != SSA_NAME)
9750 : return false;
9751 :
9752 722466 : tree_code code = gimple_assign_rhs_code (assign);
9753 722466 : if (code != ARRAY_REF
9754 722466 : && code != BIT_FIELD_REF
9755 722466 : && code != INDIRECT_REF
9756 501268 : && code != COMPONENT_REF
9757 501268 : && code != IMAGPART_EXPR
9758 361846 : && code != REALPART_EXPR
9759 361846 : && code != MEM_REF
9760 289 : && TREE_CODE_CLASS (code) != tcc_declaration)
9761 : return false;
9762 : }
9763 : else
9764 : {
9765 1458578 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
9766 4433 : if (!call || !gimple_call_internal_p (call))
9767 : return false;
9768 :
9769 4433 : internal_fn ifn = gimple_call_internal_fn (call);
9770 4433 : if (!internal_load_fn_p (ifn))
9771 : return false;
9772 :
9773 3091 : scalar_dest = gimple_call_lhs (call);
9774 3091 : if (!scalar_dest)
9775 : return false;
9776 :
9777 3091 : mask_index = internal_fn_mask_index (ifn);
9778 3091 : if (mask_index >= 0)
9779 3091 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
9780 3091 : if (mask_index >= 0
9781 3091 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
9782 : &mask_node, &mask_dt, &mask_vectype))
9783 : return false;
9784 :
9785 3091 : els_index = internal_fn_else_index (ifn);
9786 3091 : if (els_index >= 0)
9787 3091 : els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
9788 3091 : if (els_index >= 0
9789 3091 : && !vect_is_simple_use (vinfo, slp_node, els_index,
9790 : &els, &els_op, &els_dt, &els_vectype))
9791 : return false;
9792 : }
9793 :
9794 725490 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9795 725490 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9796 :
9797 725490 : if (loop_vinfo)
9798 : {
9799 506147 : loop = LOOP_VINFO_LOOP (loop_vinfo);
9800 506147 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
9801 506147 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9802 : }
9803 : else
9804 : vf = 1;
9805 :
9806 725490 : vec_num = vect_get_num_copies (vinfo, slp_node);
9807 :
9808 : /* FORNOW. This restriction should be relaxed. */
9809 725490 : if (nested_in_vect_loop && vec_num > 1)
9810 : {
9811 316 : if (dump_enabled_p ())
9812 66 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9813 : "multiple types in nested loop.\n");
9814 316 : return false;
9815 : }
9816 :
9817 725174 : elem_type = TREE_TYPE (vectype);
9818 725174 : mode = TYPE_MODE (vectype);
9819 :
9820 : /* FORNOW. In some cases can vectorize even if data-type not supported
9821 : (e.g. - data copies). */
9822 725174 : if (!can_implement_p (mov_optab, mode))
9823 : {
9824 0 : if (dump_enabled_p ())
9825 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9826 : "Aligned load, but unsupported type.\n");
9827 0 : return false;
9828 : }
9829 :
9830 : /* Check if the load is a part of an interleaving chain. */
9831 725174 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
9832 : {
9833 314004 : grouped_load = true;
9834 : /* FORNOW */
9835 314004 : gcc_assert (!nested_in_vect_loop);
9836 314004 : gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
9837 :
9838 314004 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
9839 314004 : group_size = DR_GROUP_SIZE (first_stmt_info);
9840 :
9841 : /* Invalidate assumptions made by dependence analysis when vectorization
9842 : on the unrolled body effectively re-orders stmts. */
9843 314004 : if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
9844 314004 : && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
9845 : STMT_VINFO_MIN_NEG_DIST (stmt_info)))
9846 : {
9847 12 : if (dump_enabled_p ())
9848 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9849 : "cannot perform implicit CSE when performing "
9850 : "group loads with negative dependence distance\n");
9851 12 : return false;
9852 : }
9853 : }
9854 : else
9855 : group_size = 1;
9856 :
9857 725162 : vect_load_store_data _ls_data{};
9858 725162 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
9859 725162 : if (cost_vec
9860 725162 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
9861 : VLS_LOAD, &ls))
9862 : return false;
9863 : /* Temporary aliases to analysis data, should not be modified through
9864 : these. */
9865 614777 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
9866 614777 : const dr_alignment_support alignment_support_scheme
9867 : = ls.alignment_support_scheme;
9868 614777 : const int misalignment = ls.misalignment;
9869 614777 : const poly_int64 poffset = ls.poffset;
9870 614777 : const vec<int> &elsvals = ls.elsvals;
9871 :
9872 614777 : int maskload_elsval = 0;
9873 614777 : bool need_zeroing = false;
9874 :
9875 : /* We might need to explicitly zero inactive elements if there are
9876 : padding bits in the type that might leak otherwise.
9877 : Refer to PR115336. */
9878 614777 : tree scalar_type = TREE_TYPE (scalar_dest);
9879 614777 : bool type_mode_padding_p
9880 1229554 : = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
9881 :
9882 614777 : if (slp_node->ldst_lanes
9883 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
9884 : {
9885 0 : if (dump_enabled_p ())
9886 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9887 : "discovered load-lane but cannot use it.\n");
9888 0 : return false;
9889 : }
9890 :
9891 614777 : if (mask_node)
9892 : {
9893 2961 : if (memory_access_type == VMAT_CONTIGUOUS)
9894 : {
9895 2080 : machine_mode vec_mode = TYPE_MODE (vectype);
9896 721 : if (!VECTOR_MODE_P (vec_mode)
9897 4160 : || !can_vec_mask_load_store_p (vec_mode,
9898 2080 : TYPE_MODE (mask_vectype),
9899 : true, NULL, &ls.elsvals))
9900 351 : return false;
9901 : }
9902 881 : else if (memory_access_type == VMAT_ELEMENTWISE
9903 881 : || memory_access_type == VMAT_STRIDED_SLP)
9904 : {
9905 0 : if (dump_enabled_p ())
9906 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9907 : "unsupported masked strided access.\n");
9908 0 : return false;
9909 : }
9910 881 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
9911 881 : && !mat_gather_scatter_p (memory_access_type))
9912 : {
9913 62 : if (dump_enabled_p ())
9914 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9915 : "unsupported access type for masked load.\n");
9916 62 : return false;
9917 : }
9918 819 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
9919 : {
9920 488 : if (dump_enabled_p ())
9921 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9922 : "unsupported masked emulated gather.\n");
9923 488 : return false;
9924 : }
9925 : }
9926 :
9927 613876 : bool costing_p = cost_vec;
9928 :
9929 613876 : if (costing_p) /* transformation not required. */
9930 : {
9931 445848 : if (loop_vinfo
9932 320694 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
9933 216930 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
9934 : VLS_LOAD, group_size, &ls,
9935 : mask_node, &ls.elsvals);
9936 :
9937 : /* If the type needs padding we must zero inactive elements.
9938 : Check if we can do that with a VEC_COND_EXPR and store the
9939 : elsval we choose in MASKLOAD_ELSVAL. */
9940 445848 : if (ls.elsvals.length ()
9941 60177 : && type_mode_padding_p
9942 7 : && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
9943 60177 : && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
9944 : {
9945 0 : if (dump_enabled_p ())
9946 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9947 : "cannot zero inactive elements.\n");
9948 0 : return false;
9949 : }
9950 :
9951 445848 : if (mask_node
9952 445848 : && !vect_maybe_update_slp_op_vectype (mask_node,
9953 : mask_vectype))
9954 : {
9955 0 : if (dump_enabled_p ())
9956 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9957 : "incompatible vector types for invariants\n");
9958 0 : return false;
9959 : }
9960 :
9961 445848 : if (dump_enabled_p ()
9962 25504 : && memory_access_type != VMAT_ELEMENTWISE
9963 25393 : && !mat_gather_scatter_p (memory_access_type)
9964 25078 : && memory_access_type != VMAT_STRIDED_SLP
9965 25078 : && memory_access_type != VMAT_INVARIANT
9966 470001 : && alignment_support_scheme != dr_aligned)
9967 9945 : dump_printf_loc (MSG_NOTE, vect_location,
9968 : "Vectorizing an unaligned access.\n");
9969 :
9970 445848 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
9971 0 : vinfo->any_known_not_updated_vssa = true;
9972 : }
9973 :
9974 : /* For now just use the first available else value.
9975 : get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
9976 : select it here if it is supported. */
9977 613876 : if (elsvals.length ())
9978 83338 : maskload_elsval = *elsvals.begin ();
9979 :
9980 613876 : if (dump_enabled_p () && !costing_p)
9981 16694 : dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
9982 :
9983 : /* Transform. */
9984 :
9985 613876 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
9986 613876 : ensure_base_align (dr_info);
9987 :
9988 613876 : if (memory_access_type == VMAT_INVARIANT)
9989 : {
9990 4132 : gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
9991 : /* If we have versioned for aliasing or the loop doesn't
9992 : have any data dependencies that would preclude this,
9993 : then we are sure this is a loop invariant load and
9994 : thus we can insert it on the preheader edge.
9995 : TODO: hoist_defs_of_uses should ideally be computed
9996 : once at analysis time, remembered and used in the
9997 : transform time. */
9998 8264 : bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
9999 4132 : && !nested_in_vect_loop);
10000 :
10001 4132 : bool uniform_p = true;
10002 17306 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10003 : {
10004 : /* It is unsafe to hoist a conditional load over the conditions that
10005 : make it valid. When early break this means that any invariant load
10006 : can't be hoisted unless it's in the loop header or if we know
10007 : something else has verified the load is valid to do. Alignment
10008 : peeling would do this since getting through the prologue means the
10009 : load was done at least once and so the vector main body is free to
10010 : hoist it. However today GCC will hoist the load above the PFA
10011 : loop. As such that makes it still invalid and so we can't allow it
10012 : today. */
10013 4910 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10014 1086 : && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
10015 5964 : && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
10016 1054 : != loop->header)
10017 : {
10018 954 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
10019 954 : && dump_enabled_p ())
10020 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10021 : "not hoisting invariant load due to early break"
10022 : "constraints\n");
10023 948 : else if (dump_enabled_p ())
10024 16 : dump_printf_loc (MSG_NOTE, vect_location,
10025 : "not hoisting invariant load due to early break"
10026 : "constraints\n");
10027 : hoist_p = false;
10028 : }
10029 :
10030 3956 : hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
10031 4910 : if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
10032 279 : uniform_p = false;
10033 : }
10034 4132 : if (costing_p)
10035 : {
10036 3301 : if (!uniform_p && (!hoist_p || !vf.is_constant ()))
10037 : {
10038 0 : if (dump_enabled_p ())
10039 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10040 : "not vectorizing non-uniform invariant "
10041 : "load\n");
10042 0 : return false;
10043 : }
10044 1469 : enum vect_cost_model_location cost_loc
10045 3301 : = hoist_p ? vect_prologue : vect_body;
10046 3301 : unsigned int cost = record_stmt_cost (cost_vec, 1, scalar_load,
10047 : slp_node, 0, cost_loc);
10048 3301 : cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
10049 : slp_node, 0, cost_loc);
10050 3301 : unsigned int prologue_cost = hoist_p ? cost : 0;
10051 1469 : unsigned int inside_cost = hoist_p ? 0 : cost;
10052 3301 : if (dump_enabled_p ())
10053 538 : dump_printf_loc (MSG_NOTE, vect_location,
10054 : "vect_model_load_cost: inside_cost = %d, "
10055 : "prologue_cost = %d .\n",
10056 : inside_cost, prologue_cost);
10057 3301 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10058 3301 : slp_node->data = new vect_load_store_data (std::move (ls));
10059 3301 : return true;
10060 : }
10061 831 : if (hoist_p)
10062 : {
10063 : /* ??? For non-uniform lanes there could be still duplicates.
10064 : We're leaving those to post-vectorizer CSE for the moment. */
10065 634 : auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
10066 2043 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10067 : {
10068 724 : gassign *stmt = as_a <gassign *> (sinfo->stmt);
10069 724 : if (dump_enabled_p ())
10070 352 : dump_printf_loc (MSG_NOTE, vect_location,
10071 : "hoisting out of the vectorized loop: %G",
10072 : (gimple *) stmt);
10073 724 : scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
10074 724 : tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
10075 724 : edge pe = loop_preheader_edge (loop);
10076 724 : gphi *vphi = get_virtual_phi (loop->header);
10077 724 : tree vuse;
10078 724 : if (vphi)
10079 718 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
10080 : else
10081 6 : vuse = gimple_vuse (gsi_stmt (*gsi));
10082 724 : gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
10083 724 : gimple_set_vuse (new_stmt, vuse);
10084 724 : gsi_insert_on_edge_immediate (pe, new_stmt);
10085 724 : hoist_defs_of_uses (new_stmt, loop, true);
10086 724 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
10087 724 : TREE_TYPE (scalar_dest)))
10088 : {
10089 14 : tree tem = make_ssa_name (TREE_TYPE (vectype));
10090 14 : new_stmt = gimple_build_assign (tem,
10091 : NOP_EXPR, scalar_dest);
10092 14 : gsi_insert_on_edge_immediate (pe, new_stmt);
10093 14 : scalar_dest = tem;
10094 : }
10095 724 : scalar_defs.quick_push (scalar_dest);
10096 724 : if (uniform_p)
10097 : break;
10098 : }
10099 634 : if (!uniform_p)
10100 : {
10101 51 : unsigned const_nunits
10102 51 : = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10103 116 : for (j = 0; j < (int) vec_num; ++j)
10104 : {
10105 65 : vec<constructor_elt, va_gc> *v = NULL;
10106 65 : vec_safe_reserve (v, const_nunits, true);
10107 369 : for (unsigned i = 0; i < const_nunits; ++i)
10108 : {
10109 304 : unsigned def_idx
10110 304 : = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
10111 304 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
10112 : scalar_defs[def_idx]);
10113 : }
10114 65 : scalar_dest = build_constructor (vectype, v);
10115 65 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10116 : vectype, NULL);
10117 65 : slp_node->push_vec_def (new_temp);
10118 : }
10119 51 : return true;
10120 : }
10121 583 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10122 : vectype, NULL);
10123 634 : }
10124 : else
10125 : {
10126 197 : gcc_assert (uniform_p);
10127 197 : gimple_stmt_iterator gsi2 = *gsi;
10128 197 : gsi_next (&gsi2);
10129 197 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10130 : vectype, &gsi2);
10131 : }
10132 1636 : for (j = 0; j < (int) vec_num; ++j)
10133 856 : slp_node->push_vec_def (new_temp);
10134 : return true;
10135 : }
10136 :
10137 609744 : if (memory_access_type == VMAT_ELEMENTWISE
10138 609744 : || memory_access_type == VMAT_STRIDED_SLP)
10139 : {
10140 23510 : gimple_stmt_iterator incr_gsi;
10141 23510 : bool insert_after;
10142 23510 : tree offvar = NULL_TREE;
10143 23510 : tree ivstep;
10144 23510 : tree running_off;
10145 23510 : vec<constructor_elt, va_gc> *v = NULL;
10146 23510 : tree stride_base, stride_step = NULL_TREE, alias_off;
10147 : /* Checked by get_load_store_type. */
10148 23510 : unsigned int const_nunits = nunits.to_constant ();
10149 23510 : unsigned HOST_WIDE_INT cst_offset = 0;
10150 23510 : tree dr_offset;
10151 23510 : unsigned int inside_cost = 0;
10152 :
10153 23510 : gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
10154 23510 : gcc_assert (!nested_in_vect_loop);
10155 :
10156 23510 : if (grouped_load)
10157 : {
10158 : /* If we elided a consecutive load permutation, don't
10159 : use the original first statement (which could be elided)
10160 : but the one the load permutation starts with.
10161 : This ensures the stride_base below is correct. */
10162 10723 : if (!ls.subchain_p)
10163 10679 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10164 : else
10165 44 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10166 10723 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10167 10723 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10168 : }
10169 : else
10170 : {
10171 12787 : first_stmt_info = stmt_info;
10172 12787 : first_dr_info = dr_info;
10173 12787 : ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
10174 : }
10175 :
10176 23510 : if (grouped_load)
10177 : {
10178 10723 : if (memory_access_type == VMAT_STRIDED_SLP)
10179 : {
10180 : /* If we elided a consecutive load permutation, adjust
10181 : the group size here. */
10182 4217 : if (!ls.subchain_p)
10183 4173 : group_size = DR_GROUP_SIZE (first_stmt_info);
10184 : else
10185 44 : group_size = SLP_TREE_LANES (slp_node);
10186 : }
10187 : else /* VMAT_ELEMENTWISE */
10188 6506 : group_size = SLP_TREE_LANES (slp_node);
10189 : }
10190 : else
10191 : group_size = 1;
10192 :
10193 23510 : if (!costing_p)
10194 : {
10195 3428 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
10196 3428 : stride_base = fold_build_pointer_plus (
10197 : DR_BASE_ADDRESS (first_dr_info->dr),
10198 : size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
10199 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
10200 3428 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
10201 :
10202 : /* For a load with loop-invariant (but other than power-of-2)
10203 : stride (i.e. not a grouped access) like so:
10204 :
10205 : for (i = 0; i < n; i += stride)
10206 : ... = array[i];
10207 :
10208 : we generate a new induction variable and new accesses to
10209 : form a new vector (or vectors, depending on ncopies):
10210 :
10211 : for (j = 0; ; j += VF*stride)
10212 : tmp1 = array[j];
10213 : tmp2 = array[j + stride];
10214 : ...
10215 : vectemp = {tmp1, tmp2, ...}
10216 : */
10217 :
10218 3428 : tree increment = fold_convert (TREE_TYPE (stride_step),
10219 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
10220 3428 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
10221 : stride_step, increment);
10222 :
10223 3428 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
10224 :
10225 3428 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
10226 3428 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10227 3428 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
10228 : else
10229 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
10230 : true, NULL_TREE, true,
10231 : GSI_SAME_STMT);
10232 3428 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
10233 : insert_after, &offvar, NULL, true);
10234 :
10235 3428 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
10236 : }
10237 :
10238 23510 : running_off = offvar;
10239 23510 : alias_off = build_int_cst (ref_type, 0);
10240 23510 : int nloads = const_nunits;
10241 23510 : int lnel = 1;
10242 23510 : tree ltype = TREE_TYPE (vectype);
10243 23510 : tree lvectype = vectype;
10244 23510 : auto_vec<tree> dr_chain;
10245 : /* ??? Modify local copies of alignment_support_scheme and
10246 : misalignment, but this part of analysis should be done
10247 : earlier and remembered, likewise the chosen load mode. */
10248 23510 : const dr_alignment_support tem = alignment_support_scheme;
10249 23510 : dr_alignment_support alignment_support_scheme = tem;
10250 23510 : const int tem2 = misalignment;
10251 23510 : int misalignment = tem2;
10252 23510 : if (memory_access_type == VMAT_STRIDED_SLP)
10253 : {
10254 17004 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
10255 : /* Use the target vector type if the group size is a multiple
10256 : of it. */
10257 17004 : if (n == const_nunits)
10258 : {
10259 2247 : int mis_align = dr_misalignment (first_dr_info, vectype);
10260 : /* With VF > 1 we advance the DR by step, if that is constant
10261 : and only aligned when performed VF times, DR alignment
10262 : analysis can analyze this as aligned since it assumes
10263 : contiguous accesses. But that is not how we code generate
10264 : here, so adjust for this. */
10265 2247 : if (maybe_gt (vf, 1u)
10266 3613 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10267 3401 : DR_TARGET_ALIGNMENT (first_dr_info)))
10268 212 : mis_align = -1;
10269 2247 : dr_alignment_support dr_align
10270 2247 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
10271 : mis_align);
10272 2247 : if (dr_align == dr_aligned
10273 2247 : || dr_align == dr_unaligned_supported)
10274 : {
10275 17004 : nloads = 1;
10276 17004 : lnel = const_nunits;
10277 17004 : ltype = vectype;
10278 17004 : alignment_support_scheme = dr_align;
10279 17004 : misalignment = mis_align;
10280 : }
10281 : }
10282 : /* Else use the biggest vector we can load the group without
10283 : accessing excess elements. */
10284 14757 : else if (n > 1)
10285 : {
10286 1965 : tree ptype;
10287 1965 : tree vtype
10288 1965 : = vector_vector_composition_type (vectype, const_nunits / n,
10289 : &ptype);
10290 1965 : if (vtype != NULL_TREE)
10291 : {
10292 1927 : dr_alignment_support dr_align;
10293 1927 : int mis_align = 0;
10294 1927 : if (VECTOR_TYPE_P (ptype))
10295 : {
10296 1005 : mis_align = dr_misalignment (first_dr_info, ptype);
10297 1005 : if (maybe_gt (vf, 1u)
10298 1980 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10299 1011 : DR_TARGET_ALIGNMENT (first_dr_info)))
10300 969 : mis_align = -1;
10301 1005 : dr_align
10302 1005 : = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
10303 : mis_align);
10304 : }
10305 : else
10306 : dr_align = dr_unaligned_supported;
10307 1927 : if (dr_align == dr_aligned
10308 1927 : || dr_align == dr_unaligned_supported)
10309 : {
10310 1927 : nloads = const_nunits / n;
10311 1927 : lnel = n;
10312 1927 : lvectype = vtype;
10313 1927 : ltype = ptype;
10314 1927 : alignment_support_scheme = dr_align;
10315 1927 : misalignment = mis_align;
10316 : }
10317 : }
10318 : }
10319 17004 : unsigned align;
10320 17004 : if (alignment_support_scheme == dr_aligned)
10321 20 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
10322 : else
10323 16984 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
10324 : /* Alignment is at most the access size if we do multiple loads. */
10325 17004 : if (nloads > 1)
10326 14757 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
10327 17004 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
10328 : }
10329 :
10330 23510 : if (costing_p)
10331 : {
10332 : /* Record the composition type for target access during costing. */
10333 20082 : ls.ls_type = lvectype;
10334 20082 : ls.ls_eltype = ltype;
10335 : }
10336 : else
10337 3428 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
10338 :
10339 : /* For SLP permutation support we need to load the whole group,
10340 : not only the number of vector stmts the permutation result
10341 : fits in. */
10342 23510 : int ncopies;
10343 23510 : if (ls.slp_perm)
10344 : {
10345 2869 : gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
10346 : /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10347 : variable VF. */
10348 2869 : unsigned int const_vf = vf.to_constant ();
10349 2869 : ncopies = CEIL (group_size * const_vf, const_nunits);
10350 2869 : dr_chain.create (ncopies);
10351 : }
10352 : else
10353 : ncopies = vec_num;
10354 :
10355 23510 : unsigned int group_el = 0;
10356 23510 : unsigned HOST_WIDE_INT
10357 23510 : elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
10358 23510 : unsigned int n_groups = 0;
10359 : /* For costing some adjacent vector loads, we'd like to cost with
10360 : the total number of them once instead of cost each one by one. */
10361 23510 : unsigned int n_adjacent_loads = 0;
10362 56258 : for (j = 0; j < ncopies; j++)
10363 : {
10364 32748 : if (nloads > 1 && !costing_p)
10365 3143 : vec_alloc (v, nloads);
10366 : gimple *new_stmt = NULL;
10367 137879 : for (i = 0; i < nloads; i++)
10368 : {
10369 105131 : if (costing_p)
10370 : {
10371 : /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10372 : avoid ICE, see PR110776. */
10373 95062 : if (VECTOR_TYPE_P (ltype)
10374 5822 : && memory_access_type != VMAT_ELEMENTWISE)
10375 5822 : n_adjacent_loads++;
10376 : else
10377 89240 : inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
10378 : slp_node, 0, vect_body);
10379 95062 : continue;
10380 : }
10381 10069 : unsigned int load_el = group_el;
10382 : /* For elementwise accesses apply a load permutation directly. */
10383 10069 : if (memory_access_type == VMAT_ELEMENTWISE
10384 10069 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10385 2018 : load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
10386 10069 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
10387 10069 : load_el * elsz + cst_offset);
10388 10069 : tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
10389 10069 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
10390 10069 : new_temp = make_ssa_name (ltype);
10391 10069 : new_stmt = gimple_build_assign (new_temp, data_ref);
10392 10069 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
10393 10069 : if (nloads > 1)
10394 8462 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
10395 :
10396 10069 : group_el += lnel;
10397 10069 : if (group_el == group_size)
10398 : {
10399 9722 : n_groups++;
10400 : /* When doing SLP make sure to not load elements from
10401 : the next vector iteration, those will not be accessed
10402 : so just use the last element again. See PR107451. */
10403 9722 : if (known_lt (n_groups, vf))
10404 : {
10405 6274 : tree newoff = copy_ssa_name (running_off);
10406 6274 : gimple *incr
10407 6274 : = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
10408 : running_off, stride_step);
10409 6274 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
10410 6274 : running_off = newoff;
10411 : }
10412 : group_el = 0;
10413 : }
10414 : }
10415 :
10416 32748 : if (nloads > 1)
10417 : {
10418 24011 : if (costing_p)
10419 20868 : inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
10420 : slp_node, 0, vect_body);
10421 : else
10422 : {
10423 3143 : tree vec_inv = build_constructor (lvectype, v);
10424 3143 : new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
10425 : lvectype, gsi);
10426 3143 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
10427 3143 : if (lvectype != vectype)
10428 : {
10429 398 : new_stmt
10430 398 : = gimple_build_assign (make_ssa_name (vectype),
10431 : VIEW_CONVERT_EXPR,
10432 : build1 (VIEW_CONVERT_EXPR,
10433 : vectype, new_temp));
10434 398 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10435 : gsi);
10436 : }
10437 : }
10438 : }
10439 8737 : else if (!costing_p && ltype != vectype)
10440 : {
10441 1588 : new_stmt = gimple_build_assign (make_ssa_name (vectype),
10442 : VIEW_CONVERT_EXPR,
10443 : build1 (VIEW_CONVERT_EXPR,
10444 : vectype, new_temp));
10445 1588 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10446 : gsi);
10447 : }
10448 :
10449 32748 : if (!costing_p)
10450 : {
10451 4750 : if (ls.slp_perm)
10452 1682 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
10453 : else
10454 3068 : slp_node->push_vec_def (new_stmt);
10455 : }
10456 : }
10457 23510 : if (ls.slp_perm)
10458 : {
10459 2869 : if (costing_p)
10460 : {
10461 2076 : gcc_assert (ls.n_perms != -1U);
10462 2076 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
10463 : slp_node, 0, vect_body);
10464 : }
10465 : else
10466 : {
10467 793 : unsigned n_perms2;
10468 793 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
10469 : false, &n_perms2);
10470 793 : gcc_assert (ls.n_perms == n_perms2);
10471 : }
10472 : }
10473 :
10474 23510 : if (costing_p)
10475 : {
10476 20082 : if (n_adjacent_loads > 0)
10477 2152 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10478 : alignment_support_scheme, misalignment, false,
10479 : &inside_cost, nullptr, cost_vec, cost_vec,
10480 : true);
10481 20082 : if (dump_enabled_p ())
10482 498 : dump_printf_loc (MSG_NOTE, vect_location,
10483 : "vect_model_load_cost: inside_cost = %u, "
10484 : "prologue_cost = 0 .\n",
10485 : inside_cost);
10486 20082 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10487 20082 : slp_node->data = new vect_load_store_data (std::move (ls));
10488 : }
10489 :
10490 23510 : return true;
10491 23510 : }
10492 :
10493 586234 : if (mat_gather_scatter_p (memory_access_type)
10494 586234 : && !ls.ls_type)
10495 : grouped_load = false;
10496 :
10497 583243 : if (grouped_load
10498 586234 : || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10499 : {
10500 266938 : if (grouped_load)
10501 : {
10502 266496 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10503 266496 : group_size = DR_GROUP_SIZE (first_stmt_info);
10504 : }
10505 : else
10506 : {
10507 : first_stmt_info = stmt_info;
10508 : group_size = 1;
10509 : }
10510 : /* For SLP vectorization we directly vectorize a subchain
10511 : without permutation. */
10512 266938 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10513 213094 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10514 : /* For BB vectorization always use the first stmt to base
10515 : the data ref pointer on. */
10516 266938 : if (bb_vinfo)
10517 212287 : first_stmt_info_for_drptr
10518 212287 : = vect_find_first_scalar_stmt_in_slp (slp_node);
10519 :
10520 266938 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10521 266938 : group_gap_adj = 0;
10522 :
10523 : /* VEC_NUM is the number of vect stmts to be created for this group. */
10524 266938 : grouped_load = false;
10525 : /* If an SLP permutation is from N elements to N elements,
10526 : and if one vector holds a whole number of N, we can load
10527 : the inputs to the permutation in the same way as an
10528 : unpermuted sequence. In other cases we need to load the
10529 : whole group, not only the number of vector stmts the
10530 : permutation result fits in. */
10531 266938 : unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
10532 266938 : if (nested_in_vect_loop)
10533 : /* We do not support grouped accesses in a nested loop,
10534 : instead the access is contiguous but it might be
10535 : permuted. No gap adjustment is needed though. */
10536 : ;
10537 266936 : else if (ls.slp_perm
10538 266936 : && (group_size != scalar_lanes
10539 11302 : || !multiple_p (nunits, group_size)))
10540 : {
10541 : /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10542 : variable VF; see vect_transform_slp_perm_load. */
10543 43722 : unsigned int const_vf = vf.to_constant ();
10544 43722 : unsigned int const_nunits = nunits.to_constant ();
10545 43722 : vec_num = CEIL (group_size * const_vf, const_nunits);
10546 43722 : group_gap_adj = vf * group_size - nunits * vec_num;
10547 : }
10548 : else
10549 : {
10550 223214 : group_gap_adj = group_size - scalar_lanes;
10551 : }
10552 :
10553 266938 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10554 : }
10555 : else
10556 : {
10557 319296 : first_stmt_info = stmt_info;
10558 319296 : first_dr_info = dr_info;
10559 319296 : group_size = 1;
10560 319296 : group_gap_adj = 0;
10561 319296 : ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
10562 : }
10563 :
10564 586234 : vec_loop_masks *loop_masks
10565 373947 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10566 586234 : ? &LOOP_VINFO_MASKS (loop_vinfo)
10567 31 : : NULL);
10568 31 : vec_loop_lens *loop_lens
10569 373947 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
10570 : ? &LOOP_VINFO_LENS (loop_vinfo)
10571 0 : : NULL);
10572 :
10573 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10574 : are some difference here. We cannot enable both the lens and masks
10575 : during transform but it is allowed during analysis.
10576 : Shouldn't go with length-based approach if fully masked. */
10577 586234 : if (cost_vec == NULL)
10578 : /* The cost_vec is NULL during transform. */
10579 163769 : gcc_assert ((!loop_lens || !loop_masks));
10580 :
10581 : /* Targets with store-lane instructions must not require explicit
10582 : realignment. vect_supportable_dr_alignment always returns either
10583 : dr_aligned or dr_unaligned_supported for (non-length) masked
10584 : operations. */
10585 586234 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
10586 : && !mask_node
10587 : && !loop_masks)
10588 : || mat_gather_scatter_p (memory_access_type)
10589 : || alignment_support_scheme == dr_aligned
10590 : || alignment_support_scheme == dr_unaligned_supported);
10591 :
10592 : /* In case the vectorization factor (VF) is bigger than the number
10593 : of elements that we can fit in a vectype (nunits), we have to generate
10594 : more than one vector stmt - i.e - we need to "unroll" the
10595 : vector stmt by a factor VF/nunits. In doing so, we record a pointer
10596 : from one copy of the vector stmt to the next, in the field
10597 : STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10598 : stages to find the correct vector defs to be used when vectorizing
10599 : stmts that use the defs of the current stmt. The example below
10600 : illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10601 : need to create 4 vectorized stmts):
10602 :
10603 : before vectorization:
10604 : RELATED_STMT VEC_STMT
10605 : S1: x = memref - -
10606 : S2: z = x + 1 - -
10607 :
10608 : step 1: vectorize stmt S1:
10609 : We first create the vector stmt VS1_0, and, as usual, record a
10610 : pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10611 : Next, we create the vector stmt VS1_1, and record a pointer to
10612 : it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10613 : Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10614 : stmts and pointers:
10615 : RELATED_STMT VEC_STMT
10616 : VS1_0: vx0 = memref0 VS1_1 -
10617 : VS1_1: vx1 = memref1 VS1_2 -
10618 : VS1_2: vx2 = memref2 VS1_3 -
10619 : VS1_3: vx3 = memref3 - -
10620 : S1: x = load - VS1_0
10621 : S2: z = x + 1 - -
10622 : */
10623 :
10624 : /* If the data reference is aligned (dr_aligned) or potentially unaligned
10625 : on a target that supports unaligned accesses (dr_unaligned_supported)
10626 : we generate the following code:
10627 : p = initial_addr;
10628 : indx = 0;
10629 : loop {
10630 : p = p + indx * vectype_size;
10631 : vec_dest = *(p);
10632 : indx = indx + 1;
10633 : }
10634 :
10635 : Otherwise, the data reference is potentially unaligned on a target that
10636 : does not support unaligned accesses (dr_explicit_realign_optimized) -
10637 : then generate the following code, in which the data in each iteration is
10638 : obtained by two vector loads, one from the previous iteration, and one
10639 : from the current iteration:
10640 : p1 = initial_addr;
10641 : msq_init = *(floor(p1))
10642 : p2 = initial_addr + VS - 1;
10643 : realignment_token = call target_builtin;
10644 : indx = 0;
10645 : loop {
10646 : p2 = p2 + indx * vectype_size
10647 : lsq = *(floor(p2))
10648 : vec_dest = realign_load (msq, lsq, realignment_token)
10649 : indx = indx + 1;
10650 : msq = lsq;
10651 : } */
10652 :
10653 : /* If the misalignment remains the same throughout the execution of the
10654 : loop, we can create the init_addr and permutation mask at the loop
10655 : preheader. Otherwise, it needs to be created inside the loop.
10656 : This can only occur when vectorizing memory accesses in the inner-loop
10657 : nested within an outer-loop that is being vectorized. */
10658 :
10659 586234 : if (nested_in_vect_loop
10660 586234 : && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
10661 1234 : GET_MODE_SIZE (TYPE_MODE (vectype))))
10662 : {
10663 195 : gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
10664 : compute_in_loop = true;
10665 : }
10666 :
10667 586234 : bool diff_first_stmt_info
10668 586234 : = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
10669 :
10670 586234 : tree offset = NULL_TREE;
10671 586234 : if ((alignment_support_scheme == dr_explicit_realign_optimized
10672 586234 : || alignment_support_scheme == dr_explicit_realign)
10673 0 : && !compute_in_loop)
10674 : {
10675 : /* If we have different first_stmt_info, we can't set up realignment
10676 : here, since we can't guarantee first_stmt_info DR has been
10677 : initialized yet, use first_stmt_info_for_drptr DR by bumping the
10678 : distance from first_stmt_info DR instead as below. */
10679 0 : if (!costing_p)
10680 : {
10681 0 : if (!diff_first_stmt_info)
10682 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
10683 : &realignment_token,
10684 : alignment_support_scheme, NULL_TREE,
10685 : &at_loop);
10686 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
10687 : {
10688 0 : phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
10689 0 : offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
10690 : size_one_node);
10691 0 : gcc_assert (!first_stmt_info_for_drptr);
10692 : }
10693 : }
10694 : }
10695 : else
10696 586234 : at_loop = loop;
10697 :
10698 586234 : if (!known_eq (poffset, 0))
10699 4626 : offset = (offset
10700 4626 : ? size_binop (PLUS_EXPR, offset, size_int (poffset))
10701 4626 : : size_int (poffset));
10702 :
10703 586234 : tree dr_increment;
10704 586234 : tree dr_bump;
10705 586234 : tree vec_offset = NULL_TREE;
10706 :
10707 586234 : auto_vec<tree> vec_offsets;
10708 586234 : auto_vec<tree> vec_masks;
10709 586234 : if (mask_node && !costing_p)
10710 629 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
10711 : &vec_masks);
10712 :
10713 586234 : tree vec_mask = NULL_TREE;
10714 586234 : tree vec_els = NULL_TREE;
10715 586234 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10716 : {
10717 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
10718 :
10719 0 : gcc_assert (alignment_support_scheme == dr_aligned
10720 : || alignment_support_scheme == dr_unaligned_supported);
10721 :
10722 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
10723 0 : if (!costing_p)
10724 : {
10725 0 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
10726 : memory_access_type);
10727 0 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
10728 : memory_access_type);
10729 : }
10730 :
10731 0 : unsigned int inside_cost = 0, prologue_cost = 0;
10732 : /* For costing some adjacent vector loads, we'd like to cost with
10733 : the total number of them once instead of cost each one by one. */
10734 0 : unsigned int n_adjacent_loads = 0;
10735 0 : int ncopies = vec_num / group_size;
10736 0 : for (j = 0; j < ncopies; j++)
10737 : {
10738 0 : if (costing_p)
10739 : {
10740 : /* An IFN_LOAD_LANES will load all its vector results,
10741 : regardless of which ones we actually need. Account
10742 : for the cost of unused results. */
10743 0 : if (first_stmt_info == stmt_info)
10744 : {
10745 0 : unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
10746 0 : stmt_vec_info next_stmt_info = first_stmt_info;
10747 0 : do
10748 : {
10749 0 : gaps -= 1;
10750 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
10751 : }
10752 0 : while (next_stmt_info);
10753 0 : if (gaps)
10754 : {
10755 0 : if (dump_enabled_p ())
10756 0 : dump_printf_loc (MSG_NOTE, vect_location,
10757 : "vect_model_load_cost: %d "
10758 : "unused vectors.\n",
10759 : gaps);
10760 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
10761 : alignment_support_scheme,
10762 : misalignment, false, &inside_cost,
10763 : &prologue_cost, cost_vec, cost_vec,
10764 : true);
10765 : }
10766 : }
10767 0 : n_adjacent_loads++;
10768 0 : continue;
10769 0 : }
10770 :
10771 : /* 1. Create the vector or array pointer update chain. */
10772 0 : if (j == 0)
10773 0 : dataref_ptr
10774 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10775 : at_loop, offset, &dummy, gsi,
10776 : NULL, false, dr_increment);
10777 : else
10778 : {
10779 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
10780 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10781 : stmt_info, dr_bump);
10782 : }
10783 0 : if (mask_node)
10784 0 : vec_mask = vec_masks[j];
10785 :
10786 0 : tree vec_array = create_vector_array (vectype, group_size);
10787 :
10788 0 : tree final_mask = NULL_TREE;
10789 0 : tree final_len = NULL_TREE;
10790 0 : tree bias = NULL_TREE;
10791 0 : if (loop_masks)
10792 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10793 : ncopies, vectype, j);
10794 0 : if (vec_mask)
10795 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
10796 : vec_mask, gsi);
10797 :
10798 0 : if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
10799 : {
10800 0 : if (loop_lens)
10801 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
10802 : ncopies, vectype, j, 1, true);
10803 : else
10804 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
10805 0 : signed char biasval
10806 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
10807 0 : bias = build_int_cst (intQI_type_node, biasval);
10808 0 : if (!final_mask)
10809 : {
10810 0 : mask_vectype = truth_type_for (vectype);
10811 0 : final_mask = build_minus_one_cst (mask_vectype);
10812 : }
10813 : }
10814 :
10815 0 : if (final_mask)
10816 : {
10817 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
10818 0 : if (type_mode_padding_p
10819 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
10820 0 : need_zeroing = true;
10821 : }
10822 :
10823 0 : gcall *call;
10824 0 : if (final_len && final_mask)
10825 : {
10826 : /* Emit:
10827 : VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10828 : VEC_MASK, LEN, BIAS). */
10829 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10830 0 : tree alias_ptr = build_int_cst (ref_type, align);
10831 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
10832 : dataref_ptr, alias_ptr,
10833 : final_mask, vec_els,
10834 : final_len, bias);
10835 : }
10836 0 : else if (final_mask)
10837 : {
10838 : /* Emit:
10839 : VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10840 : VEC_MASK). */
10841 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10842 0 : tree alias_ptr = build_int_cst (ref_type, align);
10843 0 : call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
10844 : dataref_ptr, alias_ptr,
10845 : final_mask, vec_els);
10846 : }
10847 : else
10848 : {
10849 : /* Emit:
10850 : VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10851 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
10852 0 : call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
10853 : }
10854 0 : gimple_call_set_lhs (call, vec_array);
10855 0 : gimple_call_set_nothrow (call, true);
10856 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
10857 :
10858 : /* Extract each vector into an SSA_NAME. */
10859 0 : for (unsigned i = 0; i < group_size; i++)
10860 : {
10861 0 : new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
10862 : vec_array, i, need_zeroing,
10863 : final_mask);
10864 0 : slp_node->push_vec_def (new_temp);
10865 : }
10866 :
10867 : /* Record that VEC_ARRAY is now dead. */
10868 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
10869 : }
10870 :
10871 0 : if (costing_p)
10872 : {
10873 0 : if (n_adjacent_loads > 0)
10874 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10875 : alignment_support_scheme, misalignment, false,
10876 : &inside_cost, &prologue_cost, cost_vec,
10877 : cost_vec, true);
10878 0 : if (dump_enabled_p ())
10879 0 : dump_printf_loc (MSG_NOTE, vect_location,
10880 : "vect_model_load_cost: inside_cost = %u, "
10881 : "prologue_cost = %u .\n",
10882 : inside_cost, prologue_cost);
10883 0 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10884 0 : slp_node->data = new vect_load_store_data (std::move (ls));
10885 : }
10886 :
10887 0 : return true;
10888 : }
10889 :
10890 586234 : if (mat_gather_scatter_p (memory_access_type))
10891 : {
10892 2991 : gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
10893 :
10894 2991 : auto_vec<tree> dr_chain (vec_num);
10895 :
10896 : /* If we pun the original vectype the loads as well as costing, length,
10897 : etc. is performed with the new type. After loading we VIEW_CONVERT
10898 : the data to the original vectype. */
10899 2991 : tree original_vectype = vectype;
10900 2991 : if (ls.ls_type)
10901 0 : vectype = ls.ls_type;
10902 :
10903 : /* 1. Create the vector or array pointer update chain. */
10904 2991 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10905 : {
10906 2991 : aggr_type = NULL_TREE;
10907 2991 : dr_increment = NULL_TREE;
10908 2991 : if (!costing_p)
10909 764 : vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
10910 : &vec_offsets);
10911 : }
10912 : else
10913 : {
10914 0 : aggr_type = elem_type;
10915 0 : if (!costing_p)
10916 : {
10917 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
10918 : ls.strided_offset_vectype,
10919 : loop_vinfo, gsi,
10920 : &dr_increment, &dr_bump,
10921 : &vec_offset);
10922 0 : dataref_ptr
10923 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10924 : at_loop, offset, &dummy, gsi,
10925 : NULL, false, dr_increment);
10926 : }
10927 : }
10928 :
10929 : unsigned int inside_cost = 0, prologue_cost = 0;
10930 :
10931 6755 : gimple *new_stmt = NULL;
10932 6755 : for (i = 0; i < vec_num; i++)
10933 : {
10934 3764 : tree final_mask = NULL_TREE;
10935 3764 : tree final_len = NULL_TREE;
10936 3764 : tree bias = NULL_TREE;
10937 3764 : if (!costing_p)
10938 : {
10939 981 : if (mask_node)
10940 156 : vec_mask = vec_masks[i];
10941 981 : if (loop_masks)
10942 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10943 : vec_num, vectype, i);
10944 981 : if (vec_mask)
10945 156 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
10946 : final_mask, vec_mask, gsi);
10947 :
10948 981 : if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10949 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10950 : stmt_info, dr_bump);
10951 : }
10952 :
10953 : /* 2. Create the vector-load in the loop. */
10954 3764 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
10955 3764 : tree alias_align_ptr = build_int_cst (ref_type, align);
10956 3764 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
10957 : {
10958 0 : if (costing_p)
10959 : {
10960 0 : if (ls.supported_offset_vectype
10961 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
10962 : vec_offset))
10963 0 : inside_cost
10964 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
10965 : slp_node, 0, vect_body);
10966 0 : if (ls.supported_scale)
10967 0 : inside_cost
10968 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
10969 : slp_node, 0, vect_body);
10970 :
10971 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
10972 0 : inside_cost
10973 0 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
10974 : slp_node, 0, vect_body);
10975 3764 : continue;
10976 0 : }
10977 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10978 0 : vec_offset = vec_offsets[i];
10979 0 : tree zero = build_zero_cst (vectype);
10980 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
10981 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
10982 :
10983 : /* Perform the offset conversion and scaling if necessary. */
10984 0 : if (!strided
10985 0 : && (ls.supported_offset_vectype || ls.supported_scale))
10986 : {
10987 0 : gimple_seq stmts = NULL;
10988 0 : if (ls.supported_offset_vectype)
10989 0 : vec_offset = gimple_convert
10990 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
10991 0 : if (ls.supported_scale)
10992 : {
10993 : /* Only scale the vec_offset if we haven't already. */
10994 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
10995 0 : || i == 0)
10996 : {
10997 0 : tree mult_cst = build_int_cst
10998 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
10999 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
11000 0 : tree mult = build_vector_from_val
11001 0 : (TREE_TYPE (vec_offset), mult_cst);
11002 0 : vec_offset = gimple_build
11003 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
11004 : vec_offset, mult);
11005 : }
11006 0 : scale = size_int (ls.supported_scale);
11007 : }
11008 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11009 : }
11010 :
11011 0 : if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
11012 : {
11013 0 : if (loop_lens)
11014 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11015 : vec_num, vectype, i, 1, true);
11016 : else
11017 0 : final_len = build_int_cst (sizetype,
11018 0 : TYPE_VECTOR_SUBPARTS (vectype));
11019 0 : signed char biasval
11020 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11021 0 : bias = build_int_cst (intQI_type_node, biasval);
11022 0 : if (!final_mask)
11023 : {
11024 0 : mask_vectype = truth_type_for (vectype);
11025 0 : final_mask = build_minus_one_cst (mask_vectype);
11026 : }
11027 : }
11028 :
11029 0 : if (final_mask)
11030 : {
11031 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11032 0 : if (type_mode_padding_p
11033 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11034 0 : need_zeroing = true;
11035 : }
11036 :
11037 0 : gcall *call;
11038 0 : if (final_len && final_mask)
11039 : {
11040 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
11041 0 : call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
11042 : 9, dataref_ptr,
11043 : alias_align_ptr,
11044 : vec_offset, scale, zero,
11045 : final_mask, vec_els,
11046 : final_len, bias);
11047 : else
11048 : /* Non-vector offset indicates that prefer to take
11049 : MASK_LEN_STRIDED_LOAD instead of the
11050 : MASK_LEN_GATHER_LOAD with direct stride arg. */
11051 0 : call = gimple_build_call_internal
11052 0 : (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
11053 : vec_offset, zero, final_mask, vec_els, final_len,
11054 : bias);
11055 : }
11056 0 : else if (final_mask)
11057 0 : call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
11058 : 7, dataref_ptr,
11059 : alias_align_ptr,
11060 : vec_offset, scale,
11061 : zero, final_mask, vec_els);
11062 : else
11063 0 : call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
11064 : dataref_ptr,
11065 : alias_align_ptr,
11066 : vec_offset, scale, zero);
11067 0 : gimple_call_set_nothrow (call, true);
11068 0 : new_stmt = call;
11069 0 : data_ref = NULL_TREE;
11070 : }
11071 3764 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
11072 : {
11073 : /* The builtin decls path for gather is legacy, x86 only. */
11074 858 : gcc_assert (!final_len && nunits.is_constant ());
11075 858 : if (costing_p)
11076 : {
11077 572 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11078 572 : inside_cost
11079 572 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11080 : slp_node, 0, vect_body);
11081 572 : continue;
11082 572 : }
11083 286 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11084 286 : poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
11085 286 : if (known_eq (nunits, offset_nunits))
11086 : {
11087 137 : new_stmt = vect_build_one_gather_load_call
11088 137 : (vinfo, stmt_info, slp_node, vectype, gsi,
11089 137 : ls.gs.decl, dataref_ptr, vec_offsets[i],
11090 : final_mask);
11091 137 : data_ref = NULL_TREE;
11092 : }
11093 149 : else if (known_eq (nunits, offset_nunits * 2))
11094 : {
11095 : /* We have a offset vector with half the number of
11096 : lanes but the builtins will produce full vectype
11097 : data with just the lower lanes filled. */
11098 63 : new_stmt = vect_build_one_gather_load_call
11099 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11100 63 : ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
11101 : final_mask);
11102 63 : tree low = make_ssa_name (vectype);
11103 63 : gimple_set_lhs (new_stmt, low);
11104 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11105 :
11106 : /* now put upper half of final_mask in final_mask low. */
11107 63 : if (final_mask
11108 63 : && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
11109 : {
11110 11 : int count = nunits.to_constant ();
11111 11 : vec_perm_builder sel (count, count, 1);
11112 11 : sel.quick_grow (count);
11113 87 : for (int i = 0; i < count; ++i)
11114 76 : sel[i] = i | (count / 2);
11115 11 : vec_perm_indices indices (sel, 2, count);
11116 11 : tree perm_mask = vect_gen_perm_mask_checked
11117 11 : (TREE_TYPE (final_mask), indices);
11118 11 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11119 : final_mask, final_mask,
11120 : perm_mask);
11121 11 : final_mask = make_ssa_name (TREE_TYPE (final_mask));
11122 11 : gimple_set_lhs (new_stmt, final_mask);
11123 11 : vect_finish_stmt_generation (vinfo, stmt_info,
11124 : new_stmt, gsi);
11125 11 : }
11126 52 : else if (final_mask)
11127 : {
11128 24 : new_stmt = gimple_build_assign (NULL_TREE,
11129 : VEC_UNPACK_HI_EXPR,
11130 : final_mask);
11131 24 : final_mask = make_ssa_name
11132 24 : (truth_type_for (offset_vectype));
11133 24 : gimple_set_lhs (new_stmt, final_mask);
11134 24 : vect_finish_stmt_generation (vinfo, stmt_info,
11135 : new_stmt, gsi);
11136 : }
11137 :
11138 63 : new_stmt = vect_build_one_gather_load_call
11139 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11140 : ls.gs.decl, dataref_ptr,
11141 63 : vec_offsets[2 * i + 1], final_mask);
11142 63 : tree high = make_ssa_name (vectype);
11143 63 : gimple_set_lhs (new_stmt, high);
11144 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11145 :
11146 : /* compose low + high. */
11147 63 : int count = nunits.to_constant ();
11148 63 : vec_perm_builder sel (count, count, 1);
11149 63 : sel.quick_grow (count);
11150 647 : for (int i = 0; i < count; ++i)
11151 584 : sel[i] = i < count / 2 ? i : i + count / 2;
11152 63 : vec_perm_indices indices (sel, 2, count);
11153 63 : tree perm_mask
11154 63 : = vect_gen_perm_mask_checked (vectype, indices);
11155 63 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11156 : low, high, perm_mask);
11157 63 : data_ref = NULL_TREE;
11158 63 : }
11159 86 : else if (known_eq (nunits * 2, offset_nunits))
11160 : {
11161 : /* We have a offset vector with double the number of
11162 : lanes. Select the low/high part accordingly. */
11163 86 : vec_offset = vec_offsets[i / 2];
11164 86 : if (i & 1)
11165 : {
11166 43 : int count = offset_nunits.to_constant ();
11167 43 : vec_perm_builder sel (count, count, 1);
11168 43 : sel.quick_grow (count);
11169 463 : for (int i = 0; i < count; ++i)
11170 420 : sel[i] = i | (count / 2);
11171 43 : vec_perm_indices indices (sel, 2, count);
11172 43 : tree perm_mask = vect_gen_perm_mask_checked
11173 43 : (TREE_TYPE (vec_offset), indices);
11174 43 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11175 : vec_offset, vec_offset,
11176 : perm_mask);
11177 43 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
11178 43 : gimple_set_lhs (new_stmt, vec_offset);
11179 43 : vect_finish_stmt_generation (vinfo, stmt_info,
11180 : new_stmt, gsi);
11181 43 : }
11182 86 : new_stmt = vect_build_one_gather_load_call
11183 86 : (vinfo, stmt_info, slp_node, vectype, gsi,
11184 : ls.gs.decl,
11185 : dataref_ptr, vec_offset, final_mask);
11186 86 : data_ref = NULL_TREE;
11187 : }
11188 : else
11189 0 : gcc_unreachable ();
11190 : }
11191 : else
11192 : {
11193 : /* Emulated gather-scatter. */
11194 2906 : gcc_assert (!final_mask);
11195 2906 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
11196 2906 : if (costing_p)
11197 : {
11198 : /* For emulated gathers N offset vector element
11199 : offset add is consumed by the load). */
11200 2211 : inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
11201 : slp_node, 0, vect_body);
11202 : /* N scalar loads plus gathering them into a
11203 : vector. */
11204 2211 : inside_cost
11205 2211 : = record_stmt_cost (cost_vec, const_nunits, scalar_load,
11206 : slp_node, 0, vect_body);
11207 2211 : inside_cost
11208 2211 : = record_stmt_cost (cost_vec, 1, vec_construct,
11209 : slp_node, 0, vect_body);
11210 2211 : continue;
11211 : }
11212 695 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11213 695 : unsigned HOST_WIDE_INT const_offset_nunits
11214 695 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
11215 695 : vec<constructor_elt, va_gc> *ctor_elts;
11216 695 : vec_alloc (ctor_elts, const_nunits);
11217 695 : gimple_seq stmts = NULL;
11218 : /* We support offset vectors with more elements
11219 : than the data vector for now. */
11220 695 : unsigned HOST_WIDE_INT factor
11221 : = const_offset_nunits / const_nunits;
11222 695 : vec_offset = vec_offsets[i / factor];
11223 695 : unsigned elt_offset = (i % factor) * const_nunits;
11224 695 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
11225 695 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11226 695 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
11227 2817 : for (unsigned k = 0; k < const_nunits; ++k)
11228 : {
11229 2122 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
11230 : bitsize_int (k + elt_offset));
11231 6366 : tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
11232 2122 : vec_offset, TYPE_SIZE (idx_type),
11233 : boff);
11234 2122 : idx = gimple_convert (&stmts, sizetype, idx);
11235 2122 : idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
11236 2122 : tree ptr = gimple_build (&stmts, PLUS_EXPR,
11237 2122 : TREE_TYPE (dataref_ptr),
11238 : dataref_ptr, idx);
11239 2122 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
11240 2122 : tree elt = make_ssa_name (TREE_TYPE (vectype));
11241 2122 : tree ref = build2 (MEM_REF, ltype, ptr,
11242 : build_int_cst (ref_type, 0));
11243 2122 : new_stmt = gimple_build_assign (elt, ref);
11244 4244 : gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
11245 2122 : gimple_seq_add_stmt (&stmts, new_stmt);
11246 2122 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
11247 : }
11248 695 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11249 695 : new_stmt = gimple_build_assign (NULL_TREE,
11250 : build_constructor (vectype,
11251 : ctor_elts));
11252 695 : data_ref = NULL_TREE;
11253 : }
11254 :
11255 981 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11256 : /* DATA_REF is null if we've already built the statement. */
11257 981 : if (data_ref)
11258 : {
11259 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11260 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11261 : }
11262 1962 : new_temp = (need_zeroing
11263 981 : ? make_ssa_name (vectype)
11264 981 : : make_ssa_name (vec_dest, new_stmt));
11265 981 : gimple_set_lhs (new_stmt, new_temp);
11266 981 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11267 :
11268 : /* If we need to explicitly zero inactive elements emit a
11269 : VEC_COND_EXPR that does so. */
11270 981 : if (need_zeroing)
11271 : {
11272 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11273 : vectype);
11274 :
11275 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11276 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11277 : final_mask, new_temp, vec_els);
11278 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11279 0 : new_temp = new_temp2;
11280 : }
11281 :
11282 981 : if (ls.ls_type)
11283 : {
11284 0 : new_stmt = gimple_build_assign (make_ssa_name
11285 : (original_vectype),
11286 : VIEW_CONVERT_EXPR,
11287 : build1 (VIEW_CONVERT_EXPR,
11288 : original_vectype,
11289 : new_temp));
11290 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11291 : }
11292 :
11293 : /* Store vector loads in the corresponding SLP_NODE. */
11294 981 : if (!costing_p)
11295 : {
11296 981 : if (ls.slp_perm)
11297 0 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
11298 : else
11299 981 : slp_node->push_vec_def (new_stmt);
11300 : }
11301 : }
11302 :
11303 2991 : if (ls.slp_perm)
11304 : {
11305 0 : if (costing_p)
11306 : {
11307 0 : gcc_assert (ls.n_perms != -1U);
11308 0 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11309 : slp_node, 0, vect_body);
11310 : }
11311 : else
11312 : {
11313 0 : unsigned n_perms2;
11314 0 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
11315 : false, &n_perms2);
11316 0 : gcc_assert (ls.n_perms == n_perms2);
11317 : }
11318 : }
11319 :
11320 2991 : if (costing_p)
11321 : {
11322 2227 : if (dump_enabled_p ())
11323 315 : dump_printf_loc (MSG_NOTE, vect_location,
11324 : "vect_model_load_cost: inside_cost = %u, "
11325 : "prologue_cost = %u .\n",
11326 : inside_cost, prologue_cost);
11327 2227 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11328 2227 : slp_node->data = new vect_load_store_data (std::move (ls));
11329 : }
11330 2991 : return true;
11331 2991 : }
11332 :
11333 583243 : aggr_type = vectype;
11334 583243 : if (!costing_p)
11335 : {
11336 163005 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
11337 : memory_access_type);
11338 163005 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
11339 : memory_access_type);
11340 : }
11341 :
11342 583243 : poly_uint64 group_elt = 0;
11343 583243 : unsigned int inside_cost = 0, prologue_cost = 0;
11344 : /* For costing some adjacent vector loads, we'd like to cost with
11345 : the total number of them once instead of cost each one by one. */
11346 583243 : unsigned int n_adjacent_loads = 0;
11347 :
11348 : /* 1. Create the vector or array pointer update chain. */
11349 583243 : if (!costing_p)
11350 : {
11351 163005 : bool simd_lane_access_p
11352 163005 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
11353 163005 : if (simd_lane_access_p
11354 1629 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
11355 1629 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
11356 1629 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
11357 1629 : && integer_zerop (DR_INIT (first_dr_info->dr))
11358 1629 : && alias_sets_conflict_p (get_alias_set (aggr_type),
11359 1629 : get_alias_set (TREE_TYPE (ref_type)))
11360 163005 : && (alignment_support_scheme == dr_aligned
11361 1629 : || alignment_support_scheme == dr_unaligned_supported))
11362 : {
11363 1629 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
11364 1629 : dataref_offset = build_int_cst (ref_type, 0);
11365 : }
11366 161376 : else if (diff_first_stmt_info)
11367 : {
11368 3742 : dataref_ptr
11369 3742 : = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
11370 : aggr_type, at_loop, offset, &dummy,
11371 : gsi, NULL, simd_lane_access_p,
11372 : dr_increment);
11373 : /* Adjust the pointer by the difference to first_stmt. */
11374 3742 : data_reference_p ptrdr
11375 : = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
11376 3742 : tree diff = fold_convert (sizetype,
11377 : size_binop (MINUS_EXPR,
11378 : DR_INIT (first_dr_info->dr),
11379 : DR_INIT (ptrdr)));
11380 3742 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11381 : stmt_info, diff);
11382 3742 : if (alignment_support_scheme == dr_explicit_realign)
11383 : {
11384 0 : msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
11385 : vectype, gsi,
11386 : &realignment_token,
11387 : alignment_support_scheme,
11388 : dataref_ptr, &at_loop);
11389 0 : gcc_assert (!compute_in_loop);
11390 : }
11391 : }
11392 : else
11393 157634 : dataref_ptr
11394 157634 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11395 : at_loop,
11396 : offset, &dummy, gsi, NULL,
11397 : simd_lane_access_p, dr_increment);
11398 : }
11399 :
11400 583243 : auto_vec<tree> dr_chain;
11401 583243 : if (grouped_load || ls.slp_perm)
11402 53844 : dr_chain.create (vec_num);
11403 :
11404 : gimple *new_stmt = NULL;
11405 1510399 : for (i = 0; i < vec_num; i++)
11406 : {
11407 927156 : tree final_mask = NULL_TREE;
11408 927156 : tree final_len = NULL_TREE;
11409 927156 : tree bias = NULL_TREE;
11410 :
11411 927156 : if (!costing_p)
11412 : {
11413 255432 : if (mask_node)
11414 659 : vec_mask = vec_masks[i];
11415 255432 : if (loop_masks)
11416 48 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11417 : vec_num, vectype, i);
11418 255432 : if (vec_mask)
11419 659 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11420 : final_mask, vec_mask, gsi);
11421 :
11422 255432 : if (i > 0)
11423 92427 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11424 : dr_bump);
11425 : }
11426 :
11427 : /* 2. Create the vector-load in the loop. */
11428 927156 : switch (alignment_support_scheme)
11429 : {
11430 927156 : case dr_aligned:
11431 927156 : case dr_unaligned_supported:
11432 927156 : {
11433 927156 : if (costing_p)
11434 : break;
11435 :
11436 255432 : unsigned int misalign;
11437 255432 : unsigned HOST_WIDE_INT align;
11438 255432 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
11439 255432 : if (alignment_support_scheme == dr_aligned)
11440 : misalign = 0;
11441 163377 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
11442 : {
11443 123968 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
11444 123968 : misalign = 0;
11445 : }
11446 : else
11447 39409 : misalign = misalignment;
11448 255432 : if (dataref_offset == NULL_TREE
11449 253305 : && TREE_CODE (dataref_ptr) == SSA_NAME)
11450 172152 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
11451 : misalign);
11452 255432 : align = least_bit_hwi (misalign | align);
11453 :
11454 : /* Compute IFN when LOOP_LENS or final_mask valid. */
11455 255432 : machine_mode vmode = TYPE_MODE (vectype);
11456 255432 : machine_mode new_vmode = vmode;
11457 255432 : internal_fn partial_ifn = IFN_LAST;
11458 255432 : if (loop_lens)
11459 : {
11460 0 : opt_machine_mode new_ovmode
11461 0 : = get_len_load_store_mode (vmode, true, &partial_ifn);
11462 0 : new_vmode = new_ovmode.require ();
11463 0 : unsigned factor
11464 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
11465 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11466 : vec_num, vectype, i, factor, true);
11467 : }
11468 255432 : else if (final_mask)
11469 : {
11470 687 : if (!can_vec_mask_load_store_p (vmode,
11471 687 : TYPE_MODE
11472 : (TREE_TYPE (final_mask)),
11473 : true, &partial_ifn))
11474 0 : gcc_unreachable ();
11475 : }
11476 :
11477 255432 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11478 : {
11479 0 : if (!final_len)
11480 : {
11481 : /* Pass VF value to 'len' argument of
11482 : MASK_LEN_LOAD if LOOP_LENS is invalid. */
11483 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11484 : }
11485 0 : if (!final_mask)
11486 : {
11487 : /* Pass all ones value to 'mask' argument of
11488 : MASK_LEN_LOAD if final_mask is invalid. */
11489 0 : mask_vectype = truth_type_for (vectype);
11490 0 : final_mask = build_minus_one_cst (mask_vectype);
11491 : }
11492 : }
11493 255432 : if (final_len)
11494 : {
11495 0 : signed char biasval
11496 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11497 0 : bias = build_int_cst (intQI_type_node, biasval);
11498 : }
11499 :
11500 255432 : tree vec_els;
11501 :
11502 255432 : if (final_len)
11503 : {
11504 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11505 0 : gcall *call;
11506 :
11507 : /* Need conversion if the vectype is punned by VnQI. */
11508 0 : els_vectype = vectype;
11509 0 : if (vmode != new_vmode)
11510 0 : els_vectype
11511 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11512 : new_vmode);
11513 0 : vec_els = vect_get_mask_load_else (maskload_elsval,
11514 : els_vectype);
11515 :
11516 0 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11517 : {
11518 0 : if (type_mode_padding_p
11519 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11520 0 : need_zeroing = true;
11521 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
11522 : 6, dataref_ptr, ptr,
11523 : final_mask, vec_els,
11524 : final_len, bias);
11525 : }
11526 : else
11527 0 : call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
11528 : dataref_ptr, ptr,
11529 : vec_els, final_len,
11530 : bias);
11531 0 : gimple_call_set_nothrow (call, true);
11532 0 : new_stmt = call;
11533 0 : data_ref = NULL_TREE;
11534 :
11535 : /* Need conversion if it's wrapped with VnQI. */
11536 0 : if (vmode != new_vmode)
11537 : {
11538 0 : tree new_vtype
11539 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11540 : new_vmode);
11541 0 : tree var = vect_get_new_ssa_name (new_vtype,
11542 : vect_simple_var);
11543 0 : gimple_set_lhs (call, var);
11544 0 : vect_finish_stmt_generation (vinfo, stmt_info, call,
11545 : gsi);
11546 0 : tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
11547 0 : new_stmt = gimple_build_assign (vec_dest,
11548 : VIEW_CONVERT_EXPR, op);
11549 : }
11550 : }
11551 255432 : else if (final_mask)
11552 : {
11553 687 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11554 687 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11555 687 : if (type_mode_padding_p
11556 687 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11557 0 : need_zeroing = true;
11558 687 : gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
11559 : dataref_ptr, ptr,
11560 : final_mask,
11561 : vec_els);
11562 687 : gimple_call_set_nothrow (call, true);
11563 687 : new_stmt = call;
11564 687 : data_ref = NULL_TREE;
11565 : }
11566 : else
11567 : {
11568 254745 : tree ltype = vectype;
11569 254745 : tree new_vtype = NULL_TREE;
11570 254745 : unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
11571 254745 : unsigned HOST_WIDE_INT dr_size
11572 254745 : = vect_get_scalar_dr_size (first_dr_info);
11573 254745 : poly_int64 off = 0;
11574 254745 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11575 1445 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
11576 254745 : unsigned int vect_align
11577 254745 : = vect_known_alignment_in_bytes (first_dr_info, vectype,
11578 254745 : off);
11579 : /* Try to use a single smaller load when we are about
11580 : to load excess elements compared to the unrolled
11581 : scalar loop. */
11582 254745 : if (known_gt ((i + 1) * nunits,
11583 : (group_size * vf - gap)))
11584 : {
11585 6968 : poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
11586 6968 : if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
11587 : nunits))
11588 : /* DR will be unused. */
11589 : ltype = NULL_TREE;
11590 2309 : else if (known_ge (vect_align,
11591 : tree_to_poly_uint64
11592 : (TYPE_SIZE_UNIT (vectype))))
11593 : /* Aligned access to excess elements is OK if
11594 : at least one element is accessed in the
11595 : scalar loop. */
11596 : ;
11597 1918 : else if (known_gt (vect_align,
11598 : ((nunits - remain) * dr_size)))
11599 : /* Aligned access to the gap area when there's
11600 : at least one element in it is OK. */
11601 : ;
11602 : else
11603 : {
11604 : /* remain should now be > 0 and < nunits. */
11605 1915 : unsigned num;
11606 1915 : if (known_ne (remain, 0u)
11607 1915 : && constant_multiple_p (nunits, remain, &num))
11608 : {
11609 1441 : tree ptype;
11610 1441 : new_vtype
11611 1441 : = vector_vector_composition_type (vectype, num,
11612 : &ptype);
11613 1441 : if (new_vtype)
11614 1441 : ltype = ptype;
11615 : }
11616 : /* Else use multiple loads or a masked load? */
11617 : /* For loop vectorization we now should have
11618 : an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11619 : set. */
11620 1915 : if (loop_vinfo)
11621 1656 : gcc_assert (new_vtype
11622 : || LOOP_VINFO_PEELING_FOR_GAPS
11623 : (loop_vinfo));
11624 : /* But still reduce the access size to the next
11625 : required power-of-two so peeling a single
11626 : scalar iteration is sufficient. */
11627 1915 : unsigned HOST_WIDE_INT cremain;
11628 1915 : if (remain.is_constant (&cremain))
11629 : {
11630 1915 : unsigned HOST_WIDE_INT cpart_size
11631 1915 : = 1 << ceil_log2 (cremain);
11632 1915 : if (known_gt (nunits, cpart_size)
11633 1915 : && constant_multiple_p (nunits, cpart_size,
11634 : &num))
11635 : {
11636 1453 : tree ptype;
11637 1453 : new_vtype
11638 2906 : = vector_vector_composition_type (vectype,
11639 1453 : num,
11640 : &ptype);
11641 1453 : if (new_vtype)
11642 1453 : ltype = ptype;
11643 : }
11644 : }
11645 : }
11646 : }
11647 254745 : tree offset = (dataref_offset ? dataref_offset
11648 252618 : : build_int_cst (ref_type, 0));
11649 254745 : if (!ltype)
11650 : ;
11651 250086 : else if (ltype != vectype
11652 250086 : && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11653 : {
11654 25 : poly_uint64 gap_offset
11655 25 : = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
11656 25 : - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
11657 25 : tree gapcst = build_int_cstu (ref_type, gap_offset);
11658 25 : offset = size_binop (PLUS_EXPR, offset, gapcst);
11659 : }
11660 254745 : if (ltype)
11661 : {
11662 250086 : data_ref = fold_build2 (MEM_REF, ltype,
11663 : dataref_ptr, offset);
11664 250086 : if (alignment_support_scheme == dr_aligned
11665 250086 : && align >= TYPE_ALIGN_UNIT (ltype))
11666 : ;
11667 : else
11668 161721 : TREE_TYPE (data_ref)
11669 323442 : = build_aligned_type (TREE_TYPE (data_ref),
11670 : align * BITS_PER_UNIT);
11671 : }
11672 254745 : if (!ltype)
11673 4659 : data_ref = build_constructor (vectype, NULL);
11674 250086 : else if (ltype != vectype)
11675 : {
11676 1453 : vect_copy_ref_info (data_ref,
11677 1453 : DR_REF (first_dr_info->dr));
11678 1453 : tree tem = make_ssa_name (ltype);
11679 1453 : new_stmt = gimple_build_assign (tem, data_ref);
11680 1453 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11681 : gsi);
11682 1453 : data_ref = NULL;
11683 1453 : vec<constructor_elt, va_gc> *v;
11684 : /* We've computed 'num' above to statically two
11685 : or via constant_multiple_p. */
11686 1453 : unsigned num
11687 1453 : = (exact_div (tree_to_poly_uint64
11688 1453 : (TYPE_SIZE_UNIT (vectype)),
11689 : tree_to_poly_uint64
11690 1453 : (TYPE_SIZE_UNIT (ltype)))
11691 1453 : .to_constant ());
11692 1453 : vec_alloc (v, num);
11693 1453 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11694 : {
11695 62 : while (--num)
11696 62 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11697 : build_zero_cst (ltype));
11698 25 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11699 : }
11700 : else
11701 : {
11702 1428 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11703 1428 : while (--num)
11704 3210 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11705 : build_zero_cst (ltype));
11706 : }
11707 1453 : gcc_assert (new_vtype != NULL_TREE);
11708 1453 : if (new_vtype == vectype)
11709 1421 : new_stmt
11710 1421 : = gimple_build_assign (vec_dest,
11711 : build_constructor (vectype, v));
11712 : else
11713 : {
11714 32 : tree new_vname = make_ssa_name (new_vtype);
11715 32 : new_stmt
11716 32 : = gimple_build_assign (new_vname,
11717 : build_constructor (new_vtype,
11718 : v));
11719 32 : vect_finish_stmt_generation (vinfo, stmt_info,
11720 : new_stmt, gsi);
11721 32 : new_stmt
11722 32 : = gimple_build_assign (vec_dest,
11723 : build1 (VIEW_CONVERT_EXPR,
11724 : vectype, new_vname));
11725 : }
11726 : }
11727 : }
11728 : break;
11729 : }
11730 0 : case dr_explicit_realign:
11731 0 : {
11732 0 : if (costing_p)
11733 : break;
11734 0 : tree ptr, bump;
11735 :
11736 0 : tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11737 :
11738 0 : if (compute_in_loop)
11739 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
11740 : gsi, &realignment_token,
11741 : dr_explicit_realign,
11742 : dataref_ptr, NULL);
11743 :
11744 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11745 0 : ptr = copy_ssa_name (dataref_ptr);
11746 : else
11747 0 : ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
11748 : // For explicit realign the target alignment should be
11749 : // known at compile time.
11750 0 : unsigned HOST_WIDE_INT align
11751 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11752 0 : new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
11753 : build_int_cst
11754 0 : (TREE_TYPE (dataref_ptr),
11755 0 : -(HOST_WIDE_INT) align));
11756 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11757 0 : data_ref = build2 (MEM_REF, vectype,
11758 : ptr, build_int_cst (ref_type, 0));
11759 0 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11760 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11761 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11762 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11763 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11764 0 : gimple_move_vops (new_stmt, stmt_info->stmt);
11765 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11766 0 : msq = new_temp;
11767 :
11768 0 : bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
11769 0 : bump = size_binop (MINUS_EXPR, bump, size_one_node);
11770 0 : ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11771 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
11772 0 : build_int_cst (TREE_TYPE (ptr),
11773 0 : -(HOST_WIDE_INT) align));
11774 0 : if (TREE_CODE (ptr) == SSA_NAME)
11775 0 : ptr = copy_ssa_name (ptr, new_stmt);
11776 : else
11777 0 : ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
11778 0 : gimple_assign_set_lhs (new_stmt, ptr);
11779 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11780 0 : data_ref = build2 (MEM_REF, vectype,
11781 : ptr, build_int_cst (ref_type, 0));
11782 0 : break;
11783 : }
11784 0 : case dr_explicit_realign_optimized:
11785 0 : {
11786 0 : if (costing_p)
11787 : break;
11788 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11789 0 : new_temp = copy_ssa_name (dataref_ptr);
11790 : else
11791 0 : new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
11792 : // We should only be doing this if we know the target
11793 : // alignment at compile time.
11794 0 : unsigned HOST_WIDE_INT align
11795 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11796 0 : new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
11797 0 : build_int_cst (TREE_TYPE (dataref_ptr),
11798 0 : -(HOST_WIDE_INT) align));
11799 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11800 0 : data_ref = build2 (MEM_REF, vectype, new_temp,
11801 : build_int_cst (ref_type, 0));
11802 0 : break;
11803 : }
11804 0 : default:
11805 0 : gcc_unreachable ();
11806 : }
11807 :
11808 : /* One common place to cost the above vect load for different
11809 : alignment support schemes. */
11810 927156 : if (costing_p)
11811 : {
11812 : /* For the prologue cost for realign,
11813 : we only need to count it once for the whole group. */
11814 671724 : bool first_stmt_info_p = first_stmt_info == stmt_info;
11815 671724 : bool add_realign_cost = first_stmt_info_p && i == 0;
11816 671724 : if (memory_access_type == VMAT_CONTIGUOUS
11817 671724 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11818 : {
11819 : /* Leave realign cases alone to keep them simple. */
11820 671724 : if (alignment_support_scheme == dr_explicit_realign_optimized
11821 : || alignment_support_scheme == dr_explicit_realign)
11822 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
11823 : alignment_support_scheme, misalignment,
11824 : add_realign_cost, &inside_cost,
11825 : &prologue_cost, cost_vec, cost_vec,
11826 : true);
11827 : else
11828 671724 : n_adjacent_loads++;
11829 : }
11830 : }
11831 : else
11832 : {
11833 255432 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11834 : /* DATA_REF is null if we've already built the statement. */
11835 255432 : if (data_ref)
11836 : {
11837 253292 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11838 253292 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11839 : }
11840 :
11841 510864 : new_temp = (need_zeroing
11842 255432 : ? make_ssa_name (vectype)
11843 255432 : : make_ssa_name (vec_dest, new_stmt));
11844 255432 : gimple_set_lhs (new_stmt, new_temp);
11845 255432 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11846 :
11847 : /* If we need to explicitly zero inactive elements emit a
11848 : VEC_COND_EXPR that does so. */
11849 255432 : if (need_zeroing)
11850 : {
11851 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11852 : vectype);
11853 :
11854 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11855 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11856 : final_mask, new_temp, vec_els);
11857 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11858 : gsi);
11859 0 : new_temp = new_temp2;
11860 : }
11861 : }
11862 :
11863 : /* 3. Handle explicit realignment if necessary/supported.
11864 : Create in loop:
11865 : vec_dest = realign_load (msq, lsq, realignment_token) */
11866 927156 : if (!costing_p
11867 255432 : && (alignment_support_scheme == dr_explicit_realign_optimized
11868 : || alignment_support_scheme == dr_explicit_realign))
11869 : {
11870 0 : lsq = gimple_assign_lhs (new_stmt);
11871 0 : if (!realignment_token)
11872 0 : realignment_token = dataref_ptr;
11873 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11874 0 : new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
11875 : lsq, realignment_token);
11876 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11877 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11878 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11879 :
11880 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
11881 : {
11882 0 : gcc_assert (phi);
11883 0 : if (i == vec_num - 1)
11884 0 : add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
11885 : UNKNOWN_LOCATION);
11886 : msq = lsq;
11887 : }
11888 : }
11889 :
11890 927156 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11891 : {
11892 5932 : if (costing_p)
11893 4487 : inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
11894 : slp_node, 0, vect_body);
11895 : else
11896 : {
11897 1445 : tree perm_mask = perm_mask_for_reverse (vectype);
11898 1445 : new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
11899 : perm_mask, stmt_info, gsi);
11900 1445 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
11901 : }
11902 : }
11903 :
11904 : /* Collect vector loads and later create their permutation in
11905 : vect_transform_slp_perm_load. */
11906 927156 : if (!costing_p && (grouped_load || ls.slp_perm))
11907 73672 : dr_chain.quick_push (new_temp);
11908 :
11909 : /* Store vector loads in the corresponding SLP_NODE. */
11910 255432 : if (!costing_p && !ls.slp_perm)
11911 181760 : slp_node->push_vec_def (new_stmt);
11912 :
11913 : /* With SLP permutation we load the gaps as well, without
11914 : we need to skip the gaps after we manage to fully load
11915 : all elements. group_gap_adj is DR_GROUP_SIZE here. */
11916 927156 : group_elt += nunits;
11917 927156 : if (!costing_p
11918 255432 : && maybe_ne (group_gap_adj, 0U)
11919 46599 : && !ls.slp_perm
11920 948856 : && known_eq (group_elt, group_size - group_gap_adj))
11921 : {
11922 17037 : poly_wide_int bump_val
11923 17037 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11924 17037 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11925 0 : bump_val = -bump_val;
11926 17037 : tree bump = wide_int_to_tree (sizetype, bump_val);
11927 17037 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11928 : bump);
11929 17037 : group_elt = 0;
11930 17037 : }
11931 : }
11932 : /* Bump the vector pointer to account for a gap or for excess
11933 : elements loaded for a permuted SLP load. */
11934 583243 : if (!costing_p
11935 163005 : && maybe_ne (group_gap_adj, 0U)
11936 600742 : && ls.slp_perm)
11937 : {
11938 462 : poly_wide_int bump_val
11939 462 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11940 462 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11941 9 : bump_val = -bump_val;
11942 462 : tree bump = wide_int_to_tree (sizetype, bump_val);
11943 462 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11944 462 : }
11945 :
11946 583243 : if (ls.slp_perm)
11947 : {
11948 : /* For SLP we know we've seen all possible uses of dr_chain so
11949 : direct vect_transform_slp_perm_load to DCE the unused parts.
11950 : ??? This is a hack to prevent compile-time issues as seen
11951 : in PR101120 and friends. */
11952 53844 : if (costing_p)
11953 : {
11954 36666 : gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
11955 36666 : if (ls.n_perms != 0)
11956 36151 : inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11957 : slp_node, 0, vect_body);
11958 36666 : if (n_adjacent_loads > 0)
11959 36666 : n_adjacent_loads = ls.n_loads;
11960 : }
11961 : else
11962 : {
11963 17178 : unsigned n_perms2, n_loads2;
11964 17178 : bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
11965 : gsi, vf, false, &n_perms2,
11966 : &n_loads2, true);
11967 17178 : gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
11968 : }
11969 : }
11970 :
11971 583243 : if (costing_p)
11972 : {
11973 420238 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
11974 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
11975 420238 : if (n_adjacent_loads > 0)
11976 420238 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
11977 : alignment_support_scheme, misalignment, false,
11978 : &inside_cost, &prologue_cost, cost_vec, cost_vec,
11979 : true);
11980 420238 : if (dump_enabled_p ())
11981 24153 : dump_printf_loc (MSG_NOTE, vect_location,
11982 : "vect_model_load_cost: inside_cost = %u, "
11983 : "prologue_cost = %u .\n",
11984 : inside_cost, prologue_cost);
11985 420238 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11986 420238 : slp_node->data = new vect_load_store_data (std::move (ls));
11987 : }
11988 :
11989 583243 : return true;
11990 1894639 : }
11991 :
11992 : /* Function vect_is_simple_cond.
11993 :
11994 : Input:
11995 : LOOP - the loop that is being vectorized.
11996 : COND - Condition that is checked for simple use.
11997 :
11998 : Output:
11999 : *COMP_VECTYPE - the vector type for the comparison.
12000 : *DTS - The def types for the arguments of the comparison
12001 :
12002 : Returns whether a COND can be vectorized. Checks whether
12003 : condition operands are supportable using vec_is_simple_use. */
12004 :
12005 : static bool
12006 35033 : vect_is_simple_cond (tree cond, vec_info *vinfo,
12007 : slp_tree slp_node, tree *comp_vectype,
12008 : enum vect_def_type *dts, tree vectype)
12009 : {
12010 35033 : tree lhs, rhs;
12011 35033 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12012 35033 : slp_tree slp_op;
12013 :
12014 : /* Mask case. */
12015 35033 : if (TREE_CODE (cond) == SSA_NAME
12016 35033 : && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond)))
12017 : {
12018 35033 : if (!vect_is_simple_use (vinfo, slp_node, 0, &cond,
12019 : &slp_op, &dts[0], comp_vectype)
12020 35033 : || !*comp_vectype
12021 70055 : || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
12022 : return false;
12023 : return true;
12024 : }
12025 :
12026 0 : if (!COMPARISON_CLASS_P (cond))
12027 : return false;
12028 :
12029 0 : lhs = TREE_OPERAND (cond, 0);
12030 0 : rhs = TREE_OPERAND (cond, 1);
12031 :
12032 0 : if (TREE_CODE (lhs) == SSA_NAME)
12033 : {
12034 0 : if (!vect_is_simple_use (vinfo, slp_node, 0,
12035 : &lhs, &slp_op, &dts[0], &vectype1))
12036 : return false;
12037 : }
12038 0 : else if (TREE_CODE (lhs) == INTEGER_CST || TREE_CODE (lhs) == REAL_CST
12039 0 : || TREE_CODE (lhs) == FIXED_CST)
12040 0 : dts[0] = vect_constant_def;
12041 : else
12042 : return false;
12043 :
12044 0 : if (TREE_CODE (rhs) == SSA_NAME)
12045 : {
12046 0 : if (!vect_is_simple_use (vinfo, slp_node, 1,
12047 : &rhs, &slp_op, &dts[1], &vectype2))
12048 : return false;
12049 : }
12050 0 : else if (TREE_CODE (rhs) == INTEGER_CST || TREE_CODE (rhs) == REAL_CST
12051 0 : || TREE_CODE (rhs) == FIXED_CST)
12052 0 : dts[1] = vect_constant_def;
12053 : else
12054 : return false;
12055 :
12056 0 : if (vectype1 && vectype2
12057 0 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12058 0 : TYPE_VECTOR_SUBPARTS (vectype2)))
12059 0 : return false;
12060 :
12061 0 : *comp_vectype = vectype1 ? vectype1 : vectype2;
12062 : /* Invariant comparison. */
12063 0 : if (! *comp_vectype)
12064 : {
12065 0 : tree scalar_type = TREE_TYPE (lhs);
12066 0 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
12067 0 : *comp_vectype = truth_type_for (vectype);
12068 : else
12069 : {
12070 : /* If we can widen the comparison to match vectype do so. */
12071 0 : if (INTEGRAL_TYPE_P (scalar_type)
12072 0 : && !slp_node
12073 0 : && tree_int_cst_lt (TYPE_SIZE (scalar_type),
12074 0 : TYPE_SIZE (TREE_TYPE (vectype))))
12075 0 : scalar_type = build_nonstandard_integer_type
12076 0 : (vector_element_bits (vectype), TYPE_UNSIGNED (scalar_type));
12077 0 : *comp_vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
12078 : slp_node);
12079 : }
12080 : }
12081 :
12082 : return true;
12083 : }
12084 :
12085 : /* vectorizable_condition.
12086 :
12087 : Check if STMT_INFO is conditional modify expression that can be vectorized.
12088 : If COST_VEC is passed, calculate costs but don't change anything,
12089 : otherwise, vectorize STMT_INFO: create a vectorized stmt using
12090 : VEC_COND_EXPR to replace it, and insert it at GSI.
12091 :
12092 : When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12093 :
12094 : Return true if STMT_INFO is vectorizable in this way. */
12095 :
12096 : static bool
12097 698251 : vectorizable_condition (vec_info *vinfo,
12098 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12099 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12100 : {
12101 698251 : tree scalar_dest = NULL_TREE;
12102 698251 : tree vec_dest = NULL_TREE;
12103 698251 : tree cond_expr, cond_expr0 = NULL_TREE, cond_expr1 = NULL_TREE;
12104 698251 : tree then_clause, else_clause;
12105 698251 : tree comp_vectype = NULL_TREE;
12106 698251 : tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
12107 698251 : tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
12108 698251 : tree vec_compare;
12109 698251 : tree new_temp;
12110 698251 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
12111 698251 : enum vect_def_type dts[4]
12112 : = {vect_unknown_def_type, vect_unknown_def_type,
12113 : vect_unknown_def_type, vect_unknown_def_type};
12114 698251 : enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12115 698251 : int i;
12116 698251 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12117 698251 : vec<tree> vec_oprnds0 = vNULL;
12118 698251 : vec<tree> vec_oprnds1 = vNULL;
12119 698251 : vec<tree> vec_oprnds2 = vNULL;
12120 698251 : vec<tree> vec_oprnds3 = vNULL;
12121 698251 : tree vec_cmp_type;
12122 698251 : bool masked = false;
12123 :
12124 698251 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12125 : return false;
12126 :
12127 : /* Is vectorizable conditional operation? */
12128 1059312 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12129 396069 : if (!stmt)
12130 : return false;
12131 :
12132 396069 : code = gimple_assign_rhs_code (stmt);
12133 396069 : if (code != COND_EXPR)
12134 : return false;
12135 :
12136 35033 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
12137 35033 : vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
12138 35033 : bool nested_cycle_p = false;
12139 35033 : bool for_reduction = vect_is_reduction (stmt_info);
12140 35033 : if (for_reduction)
12141 : {
12142 614 : if (SLP_TREE_LANES (slp_node) > 1)
12143 : return false;
12144 : /* ??? With a reduction path we do not get at the reduction info from
12145 : every stmt, use the conservative default setting then. */
12146 694 : if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
12147 : {
12148 596 : vect_reduc_info reduc_info
12149 596 : = info_for_reduction (loop_vinfo, slp_node);
12150 596 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
12151 596 : nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12152 : stmt_info);
12153 : }
12154 : }
12155 : else
12156 : {
12157 34419 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12158 : return false;
12159 : }
12160 :
12161 35033 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12162 35033 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12163 :
12164 35033 : int vec_num = vect_get_num_copies (vinfo, slp_node);
12165 :
12166 35033 : cond_expr = gimple_assign_rhs1 (stmt);
12167 35033 : gcc_assert (! COMPARISON_CLASS_P (cond_expr));
12168 :
12169 35033 : if (!vect_is_simple_cond (cond_expr, vinfo, slp_node,
12170 : &comp_vectype, &dts[0], vectype)
12171 35033 : || !comp_vectype)
12172 : return false;
12173 :
12174 35022 : unsigned op_adjust = COMPARISON_CLASS_P (cond_expr) ? 1 : 0;
12175 35022 : slp_tree then_slp_node, else_slp_node;
12176 35022 : if (!vect_is_simple_use (vinfo, slp_node, 1 + op_adjust,
12177 : &then_clause, &then_slp_node, &dts[2], &vectype1))
12178 : return false;
12179 35022 : if (!vect_is_simple_use (vinfo, slp_node, 2 + op_adjust,
12180 : &else_clause, &else_slp_node, &dts[3], &vectype2))
12181 : return false;
12182 :
12183 35022 : if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
12184 : return false;
12185 :
12186 35022 : if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
12187 : return false;
12188 :
12189 35022 : masked = !COMPARISON_CLASS_P (cond_expr);
12190 35022 : vec_cmp_type = truth_type_for (comp_vectype);
12191 35022 : if (vec_cmp_type == NULL_TREE
12192 70044 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
12193 35022 : TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
12194 0 : return false;
12195 :
12196 35022 : cond_code = TREE_CODE (cond_expr);
12197 35022 : if (!masked)
12198 : {
12199 0 : cond_expr0 = TREE_OPERAND (cond_expr, 0);
12200 0 : cond_expr1 = TREE_OPERAND (cond_expr, 1);
12201 : }
12202 :
12203 : /* For conditional reductions, the "then" value needs to be the candidate
12204 : value calculated by this iteration while the "else" value needs to be
12205 : the result carried over from previous iterations. If the COND_EXPR
12206 : is the other way around, we need to swap it. */
12207 35022 : bool must_invert_cmp_result = false;
12208 35022 : if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
12209 : {
12210 0 : if (masked)
12211 0 : must_invert_cmp_result = true;
12212 : else
12213 : {
12214 0 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond_expr0));
12215 0 : tree_code new_code = invert_tree_comparison (cond_code, honor_nans);
12216 0 : if (new_code == ERROR_MARK)
12217 : must_invert_cmp_result = true;
12218 : else
12219 : {
12220 0 : cond_code = new_code;
12221 : /* Make sure we don't accidentally use the old condition. */
12222 0 : cond_expr = NULL_TREE;
12223 : }
12224 : }
12225 : /* ??? The vectorized operand query below doesn't allow swapping
12226 : this way for SLP. */
12227 0 : return false;
12228 : /* std::swap (then_clause, else_clause); */
12229 : }
12230 :
12231 35022 : if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype))
12232 : {
12233 : /* Boolean values may have another representation in vectors
12234 : and therefore we prefer bit operations over comparison for
12235 : them (which also works for scalar masks). We store opcodes
12236 : to use in bitop1 and bitop2. Statement is vectorized as
12237 : BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12238 : depending on bitop1 and bitop2 arity. */
12239 0 : switch (cond_code)
12240 : {
12241 : case GT_EXPR:
12242 : bitop1 = BIT_NOT_EXPR;
12243 : bitop2 = BIT_AND_EXPR;
12244 : break;
12245 0 : case GE_EXPR:
12246 0 : bitop1 = BIT_NOT_EXPR;
12247 0 : bitop2 = BIT_IOR_EXPR;
12248 0 : break;
12249 0 : case LT_EXPR:
12250 0 : bitop1 = BIT_NOT_EXPR;
12251 0 : bitop2 = BIT_AND_EXPR;
12252 0 : std::swap (cond_expr0, cond_expr1);
12253 0 : break;
12254 0 : case LE_EXPR:
12255 0 : bitop1 = BIT_NOT_EXPR;
12256 0 : bitop2 = BIT_IOR_EXPR;
12257 0 : std::swap (cond_expr0, cond_expr1);
12258 0 : break;
12259 0 : case NE_EXPR:
12260 0 : bitop1 = BIT_XOR_EXPR;
12261 0 : break;
12262 0 : case EQ_EXPR:
12263 0 : bitop1 = BIT_XOR_EXPR;
12264 0 : bitop2 = BIT_NOT_EXPR;
12265 0 : break;
12266 : default:
12267 : return false;
12268 : }
12269 : cond_code = SSA_NAME;
12270 : }
12271 :
12272 35022 : if (TREE_CODE_CLASS (cond_code) == tcc_comparison
12273 0 : && reduction_type == EXTRACT_LAST_REDUCTION
12274 35022 : && !expand_vec_cmp_expr_p (comp_vectype, vec_cmp_type, cond_code))
12275 : {
12276 0 : if (dump_enabled_p ())
12277 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12278 : "reduction comparison operation not supported.\n");
12279 0 : return false;
12280 : }
12281 :
12282 35022 : if (cost_vec)
12283 : {
12284 26423 : if (bitop1 != NOP_EXPR)
12285 : {
12286 0 : machine_mode mode = TYPE_MODE (comp_vectype);
12287 0 : optab optab;
12288 :
12289 0 : optab = optab_for_tree_code (bitop1, comp_vectype, optab_default);
12290 0 : if (!optab || !can_implement_p (optab, mode))
12291 0 : return false;
12292 :
12293 0 : if (bitop2 != NOP_EXPR)
12294 : {
12295 0 : optab = optab_for_tree_code (bitop2, comp_vectype,
12296 : optab_default);
12297 0 : if (!optab || !can_implement_p (optab, mode))
12298 0 : return false;
12299 : }
12300 : }
12301 :
12302 26423 : vect_cost_for_stmt kind = vector_stmt;
12303 26423 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12304 : /* Count one reduction-like operation per vector. */
12305 : kind = vec_to_scalar;
12306 26423 : else if ((masked && !expand_vec_cond_expr_p (vectype, comp_vectype))
12307 26423 : || (!masked
12308 0 : && (!expand_vec_cmp_expr_p (comp_vectype, vec_cmp_type,
12309 : cond_code)
12310 0 : || !expand_vec_cond_expr_p (vectype, vec_cmp_type))))
12311 14 : return false;
12312 :
12313 26409 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
12314 : comp_vectype)
12315 26409 : || (op_adjust == 1
12316 0 : && !vect_maybe_update_slp_op_vectype
12317 0 : (SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
12318 26409 : || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
12319 52818 : || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
12320 : {
12321 0 : if (dump_enabled_p ())
12322 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12323 : "incompatible vector types for invariants\n");
12324 0 : return false;
12325 : }
12326 :
12327 26409 : if (loop_vinfo && for_reduction
12328 447 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12329 : {
12330 68 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12331 : {
12332 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12333 : vectype, OPTIMIZE_FOR_SPEED))
12334 0 : vect_record_loop_len (loop_vinfo,
12335 : &LOOP_VINFO_LENS (loop_vinfo),
12336 : vec_num, vectype, 1);
12337 : else
12338 0 : vect_record_loop_mask (loop_vinfo,
12339 : &LOOP_VINFO_MASKS (loop_vinfo),
12340 : vec_num, vectype, NULL);
12341 : }
12342 : /* Extra inactive lanes should be safe for vect_nested_cycle. */
12343 68 : else if (!nested_cycle_p)
12344 : {
12345 68 : if (dump_enabled_p ())
12346 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12347 : "conditional reduction prevents the use"
12348 : " of partial vectors.\n");
12349 68 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
12350 : }
12351 : }
12352 :
12353 26409 : SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
12354 26409 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
12355 26409 : return true;
12356 : }
12357 :
12358 : /* Transform. */
12359 :
12360 : /* Handle def. */
12361 8599 : scalar_dest = gimple_assign_lhs (stmt);
12362 8599 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12363 8599 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
12364 :
12365 8599 : bool swap_cond_operands = false;
12366 :
12367 : /* See whether another part of the vectorized code applies a loop
12368 : mask to the condition, or to its inverse. */
12369 :
12370 8599 : vec_loop_masks *masks = NULL;
12371 8599 : vec_loop_lens *lens = NULL;
12372 8599 : if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12373 : {
12374 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12375 0 : lens = &LOOP_VINFO_LENS (loop_vinfo);
12376 : }
12377 8599 : else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
12378 : {
12379 3 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12380 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12381 : else
12382 : {
12383 3 : scalar_cond_masked_key cond (cond_expr, 1);
12384 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12385 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12386 : else
12387 : {
12388 3 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
12389 3 : tree_code orig_code = cond.code;
12390 3 : cond.code = invert_tree_comparison (cond.code, honor_nans);
12391 3 : if (!masked && loop_vinfo->scalar_cond_masked_set.contains (cond))
12392 : {
12393 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12394 0 : cond_code = cond.code;
12395 0 : swap_cond_operands = true;
12396 : }
12397 : else
12398 : {
12399 : /* Try the inverse of the current mask. We check if the
12400 : inverse mask is live and if so we generate a negate of
12401 : the current mask such that we still honor NaNs. */
12402 3 : cond.inverted_p = true;
12403 3 : cond.code = orig_code;
12404 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12405 : {
12406 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12407 0 : cond_code = cond.code;
12408 0 : swap_cond_operands = true;
12409 0 : must_invert_cmp_result = true;
12410 : }
12411 : }
12412 : }
12413 : }
12414 : }
12415 :
12416 : /* Handle cond expr. */
12417 8599 : if (masked)
12418 8599 : vect_get_vec_defs (vinfo, slp_node,
12419 : cond_expr, &vec_oprnds0,
12420 : then_clause, &vec_oprnds2,
12421 : reduction_type != EXTRACT_LAST_REDUCTION
12422 : ? else_clause : NULL, &vec_oprnds3);
12423 : else
12424 0 : vect_get_vec_defs (vinfo, slp_node,
12425 : cond_expr0, &vec_oprnds0,
12426 : cond_expr1, &vec_oprnds1,
12427 : then_clause, &vec_oprnds2,
12428 : reduction_type != EXTRACT_LAST_REDUCTION
12429 : ? else_clause : NULL, &vec_oprnds3);
12430 :
12431 8599 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12432 0 : vec_else_clause = else_clause;
12433 :
12434 : /* Arguments are ready. Create the new vector stmt. */
12435 20234 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
12436 : {
12437 11635 : vec_then_clause = vec_oprnds2[i];
12438 11635 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12439 11635 : vec_else_clause = vec_oprnds3[i];
12440 :
12441 11635 : if (swap_cond_operands)
12442 0 : std::swap (vec_then_clause, vec_else_clause);
12443 :
12444 11635 : if (masked)
12445 : vec_compare = vec_cond_lhs;
12446 : else
12447 : {
12448 0 : vec_cond_rhs = vec_oprnds1[i];
12449 0 : if (bitop1 == NOP_EXPR)
12450 : {
12451 0 : gimple_seq stmts = NULL;
12452 0 : vec_compare = gimple_build (&stmts, cond_code, vec_cmp_type,
12453 : vec_cond_lhs, vec_cond_rhs);
12454 0 : gsi_insert_before (gsi, stmts, GSI_SAME_STMT);
12455 : }
12456 : else
12457 : {
12458 0 : new_temp = make_ssa_name (vec_cmp_type);
12459 0 : gassign *new_stmt;
12460 0 : if (bitop1 == BIT_NOT_EXPR)
12461 0 : new_stmt = gimple_build_assign (new_temp, bitop1,
12462 : vec_cond_rhs);
12463 : else
12464 0 : new_stmt
12465 0 : = gimple_build_assign (new_temp, bitop1, vec_cond_lhs,
12466 : vec_cond_rhs);
12467 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12468 0 : if (bitop2 == NOP_EXPR)
12469 : vec_compare = new_temp;
12470 0 : else if (bitop2 == BIT_NOT_EXPR
12471 0 : && reduction_type != EXTRACT_LAST_REDUCTION)
12472 : {
12473 : /* Instead of doing ~x ? y : z do x ? z : y. */
12474 : vec_compare = new_temp;
12475 : std::swap (vec_then_clause, vec_else_clause);
12476 : }
12477 : else
12478 : {
12479 0 : vec_compare = make_ssa_name (vec_cmp_type);
12480 0 : if (bitop2 == BIT_NOT_EXPR)
12481 0 : new_stmt
12482 0 : = gimple_build_assign (vec_compare, bitop2, new_temp);
12483 : else
12484 0 : new_stmt
12485 0 : = gimple_build_assign (vec_compare, bitop2,
12486 : vec_cond_lhs, new_temp);
12487 0 : vect_finish_stmt_generation (vinfo, stmt_info,
12488 : new_stmt, gsi);
12489 : }
12490 : }
12491 : }
12492 :
12493 : /* If we decided to apply a loop mask to the result of the vector
12494 : comparison, AND the comparison with the mask now. Later passes
12495 : should then be able to reuse the AND results between multiple
12496 : vector statements.
12497 :
12498 : For example:
12499 : for (int i = 0; i < 100; ++i)
12500 : x[i] = y[i] ? z[i] : 10;
12501 :
12502 : results in following optimized GIMPLE:
12503 :
12504 : mask__35.8_43 = vect__4.7_41 != { 0, ... };
12505 : vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12506 : _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12507 : vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12508 : vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12509 : vect_iftmp.11_47, { 10, ... }>;
12510 :
12511 : instead of using a masked and unmasked forms of
12512 : vec != { 0, ... } (masked in the MASK_LOAD,
12513 : unmasked in the VEC_COND_EXPR). */
12514 :
12515 : /* Force vec_compare to be an SSA_NAME rather than a comparison,
12516 : in cases where that's necessary. */
12517 :
12518 11635 : tree len = NULL_TREE, bias = NULL_TREE;
12519 11635 : if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
12520 : {
12521 0 : if (!is_gimple_val (vec_compare))
12522 : {
12523 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12524 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12525 : vec_compare);
12526 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12527 0 : vec_compare = vec_compare_name;
12528 : }
12529 :
12530 0 : if (must_invert_cmp_result)
12531 : {
12532 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12533 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12534 : BIT_NOT_EXPR,
12535 : vec_compare);
12536 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12537 0 : vec_compare = vec_compare_name;
12538 : }
12539 :
12540 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12541 : vectype, OPTIMIZE_FOR_SPEED))
12542 : {
12543 0 : if (lens)
12544 : {
12545 : /* ??? Do we really want the adjusted LEN here? Isn't this
12546 : based on number of elements? */
12547 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens,
12548 : vec_num, vectype, i, 1, true);
12549 0 : signed char biasval
12550 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
12551 0 : bias = build_int_cst (intQI_type_node, biasval);
12552 : }
12553 : else
12554 : {
12555 0 : len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
12556 0 : bias = build_int_cst (intQI_type_node, 0);
12557 : }
12558 : }
12559 0 : if (masks)
12560 : {
12561 0 : tree loop_mask
12562 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12563 : vectype, i);
12564 0 : tree tmp2 = make_ssa_name (vec_cmp_type);
12565 0 : gassign *g
12566 0 : = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
12567 : loop_mask);
12568 0 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
12569 0 : vec_compare = tmp2;
12570 : }
12571 : }
12572 :
12573 0 : gimple *new_stmt;
12574 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12575 : {
12576 0 : gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
12577 0 : tree lhs = gimple_get_lhs (old_stmt);
12578 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12579 0 : lhs = copy_ssa_name (lhs);
12580 0 : if (len)
12581 0 : new_stmt = gimple_build_call_internal
12582 0 : (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
12583 : vec_then_clause, len, bias);
12584 : else
12585 0 : new_stmt = gimple_build_call_internal
12586 0 : (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
12587 : vec_then_clause);
12588 0 : gimple_call_set_lhs (new_stmt, lhs);
12589 0 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12590 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12591 : {
12592 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12593 0 : vec_else_clause = lhs;
12594 : }
12595 0 : else if (old_stmt == gsi_stmt (*gsi))
12596 0 : vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
12597 : else
12598 : {
12599 : /* In this case we're moving the definition to later in the
12600 : block. That doesn't matter because the only uses of the
12601 : lhs are in phi statements. */
12602 0 : gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
12603 0 : gsi_remove (&old_gsi, true);
12604 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12605 : }
12606 : }
12607 : else
12608 : {
12609 11635 : new_temp = make_ssa_name (vec_dest);
12610 11635 : new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
12611 : vec_then_clause, vec_else_clause);
12612 11635 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12613 : }
12614 11635 : slp_node->push_vec_def (new_stmt);
12615 : }
12616 :
12617 8599 : vec_oprnds0.release ();
12618 8599 : vec_oprnds1.release ();
12619 8599 : vec_oprnds2.release ();
12620 8599 : vec_oprnds3.release ();
12621 :
12622 8599 : return true;
12623 : }
12624 :
12625 : /* Helper of vectorizable_comparison.
12626 :
12627 : Check if STMT_INFO is comparison expression CODE that can be vectorized.
12628 : If COST_VEC is passed, calculate costs but don't change anything,
12629 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12630 : it at GSI.
12631 :
12632 : Return true if STMT_INFO is vectorizable in this way. */
12633 :
12634 : static bool
12635 369508 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
12636 : stmt_vec_info stmt_info, tree_code code,
12637 : gimple_stmt_iterator *gsi,
12638 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12639 : {
12640 369508 : tree lhs, rhs1, rhs2;
12641 369508 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12642 369508 : tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
12643 369508 : tree new_temp;
12644 369508 : enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
12645 369508 : poly_uint64 nunits;
12646 369508 : enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12647 369508 : int i;
12648 369508 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12649 369508 : vec<tree> vec_oprnds0 = vNULL;
12650 369508 : vec<tree> vec_oprnds1 = vNULL;
12651 369508 : tree mask_type;
12652 369508 : tree mask = NULL_TREE;
12653 :
12654 369508 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12655 : return false;
12656 :
12657 369508 : if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
12658 : return false;
12659 :
12660 168528 : mask_type = vectype;
12661 168528 : nunits = TYPE_VECTOR_SUBPARTS (vectype);
12662 :
12663 168528 : if (TREE_CODE_CLASS (code) != tcc_comparison)
12664 : return false;
12665 :
12666 166744 : slp_tree slp_rhs1, slp_rhs2;
12667 166744 : if (!vect_is_simple_use (vinfo, slp_node,
12668 : 0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
12669 : return false;
12670 :
12671 166744 : if (!vect_is_simple_use (vinfo, slp_node,
12672 : 1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
12673 : return false;
12674 :
12675 129497 : if (vectype1 && vectype2
12676 242159 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12677 75415 : TYPE_VECTOR_SUBPARTS (vectype2)))
12678 16 : return false;
12679 :
12680 166728 : vectype = vectype1 ? vectype1 : vectype2;
12681 :
12682 : /* Invariant comparison. */
12683 166728 : if (!vectype)
12684 : {
12685 32528 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
12686 32528 : if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
12687 9 : return false;
12688 : }
12689 134200 : else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
12690 : return false;
12691 :
12692 : /* Can't compare mask and non-mask types. */
12693 129481 : if (vectype1 && vectype2
12694 392300 : && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
12695 : return false;
12696 :
12697 : /* Boolean values may have another representation in vectors
12698 : and therefore we prefer bit operations over comparison for
12699 : them (which also works for scalar masks). We store opcodes
12700 : to use in bitop1 and bitop2. Statement is vectorized as
12701 : BITOP2 (rhs1 BITOP1 rhs2) or
12702 : rhs1 BITOP2 (BITOP1 rhs2)
12703 : depending on bitop1 and bitop2 arity. */
12704 166711 : bool swap_p = false;
12705 166711 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
12706 : {
12707 756 : if (code == GT_EXPR)
12708 : {
12709 : bitop1 = BIT_NOT_EXPR;
12710 : bitop2 = BIT_AND_EXPR;
12711 : }
12712 : else if (code == GE_EXPR)
12713 : {
12714 : bitop1 = BIT_NOT_EXPR;
12715 : bitop2 = BIT_IOR_EXPR;
12716 : }
12717 : else if (code == LT_EXPR)
12718 : {
12719 : bitop1 = BIT_NOT_EXPR;
12720 : bitop2 = BIT_AND_EXPR;
12721 : swap_p = true;
12722 : }
12723 : else if (code == LE_EXPR)
12724 : {
12725 : bitop1 = BIT_NOT_EXPR;
12726 : bitop2 = BIT_IOR_EXPR;
12727 : swap_p = true;
12728 : }
12729 : else
12730 : {
12731 : bitop1 = BIT_XOR_EXPR;
12732 : if (code == EQ_EXPR)
12733 : bitop2 = BIT_NOT_EXPR;
12734 : }
12735 : }
12736 :
12737 166711 : if (cost_vec)
12738 : {
12739 154103 : if (bitop1 == NOP_EXPR)
12740 : {
12741 153493 : if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
12742 : return false;
12743 : }
12744 : else
12745 : {
12746 610 : machine_mode mode = TYPE_MODE (vectype);
12747 610 : optab optab;
12748 :
12749 610 : optab = optab_for_tree_code (bitop1, vectype, optab_default);
12750 610 : if (!optab || !can_implement_p (optab, mode))
12751 0 : return false;
12752 :
12753 610 : if (bitop2 != NOP_EXPR)
12754 : {
12755 95 : optab = optab_for_tree_code (bitop2, vectype, optab_default);
12756 95 : if (!optab || !can_implement_p (optab, mode))
12757 0 : return false;
12758 : }
12759 : }
12760 :
12761 : /* Put types on constant and invariant SLP children. */
12762 145678 : if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
12763 145678 : || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
12764 : {
12765 2 : if (dump_enabled_p ())
12766 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12767 : "incompatible vector types for invariants\n");
12768 2 : return false;
12769 : }
12770 :
12771 145676 : vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
12772 : slp_node, cost_vec);
12773 145676 : return true;
12774 : }
12775 :
12776 : /* Transform. */
12777 :
12778 : /* Handle def. */
12779 12608 : lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
12780 12608 : if (lhs)
12781 12608 : mask = vect_create_destination_var (lhs, mask_type);
12782 :
12783 12608 : vect_get_vec_defs (vinfo, slp_node, rhs1, &vec_oprnds0, rhs2, &vec_oprnds1);
12784 12608 : if (swap_p)
12785 58 : std::swap (vec_oprnds0, vec_oprnds1);
12786 :
12787 : /* Arguments are ready. Create the new vector stmt. */
12788 31709 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
12789 : {
12790 19101 : gimple *new_stmt;
12791 19101 : vec_rhs2 = vec_oprnds1[i];
12792 :
12793 19101 : if (lhs)
12794 19101 : new_temp = make_ssa_name (mask);
12795 : else
12796 0 : new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
12797 19101 : if (bitop1 == NOP_EXPR)
12798 : {
12799 18945 : new_stmt = gimple_build_assign (new_temp, code,
12800 : vec_rhs1, vec_rhs2);
12801 18945 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12802 : }
12803 : else
12804 : {
12805 156 : if (bitop1 == BIT_NOT_EXPR)
12806 84 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
12807 : else
12808 72 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
12809 : vec_rhs2);
12810 156 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12811 156 : if (bitop2 != NOP_EXPR)
12812 : {
12813 84 : tree res = make_ssa_name (mask);
12814 84 : if (bitop2 == BIT_NOT_EXPR)
12815 0 : new_stmt = gimple_build_assign (res, bitop2, new_temp);
12816 : else
12817 84 : new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
12818 : new_temp);
12819 84 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12820 : }
12821 : }
12822 19101 : slp_node->push_vec_def (new_stmt);
12823 : }
12824 :
12825 12608 : vec_oprnds0.release ();
12826 12608 : vec_oprnds1.release ();
12827 :
12828 12608 : return true;
12829 : }
12830 :
12831 : /* vectorizable_comparison.
12832 :
12833 : Check if STMT_INFO is comparison expression that can be vectorized.
12834 : If COST_VEC is passed, calculate costs but don't change anything,
12835 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12836 : it at GSI.
12837 :
12838 : Return true if STMT_INFO is vectorizable in this way. */
12839 :
12840 : static bool
12841 675851 : vectorizable_comparison (vec_info *vinfo,
12842 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12843 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12844 : {
12845 675851 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12846 :
12847 675851 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12848 : return false;
12849 :
12850 675851 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12851 : return false;
12852 :
12853 887075 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12854 366879 : if (!stmt)
12855 : return false;
12856 :
12857 366879 : enum tree_code code = gimple_assign_rhs_code (stmt);
12858 366879 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12859 366879 : if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
12860 : slp_node, cost_vec))
12861 : return false;
12862 :
12863 155655 : if (cost_vec)
12864 143047 : SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
12865 :
12866 : return true;
12867 : }
12868 :
12869 : /* Check to see if the target supports any of the compare and branch optabs for
12870 : vectors with MODE as these would be required when expanding. */
12871 : static bool
12872 65330 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
12873 : {
12874 65330 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12875 65330 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12876 :
12877 : /* The vectorizer only produces vec_cbranch_any_optab directly. So only
12878 : check for support for that or vec_cbranch_any_optab when masked.
12879 : We can't produce vcond_cbranch_any directly from the vectorizer as we
12880 : want to keep gimple_cond as the GIMPLE representation. But we'll fold
12881 : it in expand. For that reason we require a backend to support the
12882 : unconditional vector cbranch optab if they support the conditional one,
12883 : which is just an optimization on the unconditional one. */
12884 65330 : if (masked_loop_p
12885 65330 : && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
12886 : != CODE_FOR_nothing)
12887 : return true;
12888 65330 : else if (len_loop_p
12889 65330 : && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
12890 : != CODE_FOR_nothing)
12891 : return true;
12892 65330 : else if (!masked_loop_p && !len_loop_p
12893 130660 : && direct_optab_handler (vec_cbranch_any_optab, mode)
12894 : != CODE_FOR_nothing)
12895 : return true;
12896 :
12897 : /* The target can implement cbranch to distinguish between boolean vector
12898 : types and data types if they don't have a different mode for both. */
12899 65330 : return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
12900 : }
12901 :
12902 : /* Determine the type to use for early break vectorization's scalar IV. If
12903 : no type is possible return false. */
12904 :
12905 : static bool
12906 2629 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
12907 : {
12908 : /* Check if we have a usable scalar IV type for vectorization. */
12909 2629 : tree iters_vf_type = sizetype;
12910 2629 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
12911 : {
12912 : /* Find the type with the minimum precision we can use
12913 : for the scalar IV. */
12914 2406 : tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
12915 :
12916 : /* Work out how many bits we need to represent the limit. */
12917 2406 : unsigned int min_ni_width
12918 2406 : = vect_min_prec_for_max_niters (loop_vinfo, 1);
12919 :
12920 : /* Check if we're using PFA, if so we need a signed IV and an
12921 : extra bit for the sign. */
12922 2406 : if (TYPE_UNSIGNED (cand_type)
12923 2406 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
12924 3972 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12925 158 : min_ni_width += 1;
12926 :
12927 2406 : if (TYPE_PRECISION (cand_type) >= min_ni_width)
12928 2333 : iters_vf_type = unsigned_type_for (cand_type);
12929 : else
12930 : {
12931 73 : opt_scalar_int_mode cmp_mode_iter;
12932 73 : tree iv_type = NULL_TREE;
12933 357 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
12934 : {
12935 357 : auto cmp_mode = cmp_mode_iter.require ();
12936 357 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
12937 357 : if (cmp_bits >= min_ni_width
12938 357 : && targetm.scalar_mode_supported_p (cmp_mode))
12939 : {
12940 73 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
12941 73 : if (iv_type)
12942 : break;
12943 : }
12944 : }
12945 :
12946 73 : if (!iv_type)
12947 : {
12948 0 : if (dump_enabled_p ())
12949 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12950 : "can't vectorize early exit because the "
12951 : "target doesn't support a scalar type wide "
12952 : "wide enough to hold niters.\n");
12953 0 : return false;
12954 : }
12955 73 : iters_vf_type = iv_type;
12956 : }
12957 : }
12958 :
12959 2629 : LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
12960 2629 : return true;
12961 : }
12962 :
12963 : /* Check to see if the current early break given in STMT_INFO is valid for
12964 : vectorization. */
12965 :
12966 : bool
12967 253416 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
12968 : gimple_stmt_iterator *gsi,
12969 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12970 : {
12971 253416 : if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
12972 : return false;
12973 :
12974 66919 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
12975 : return false;
12976 :
12977 66919 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
12978 : return false;
12979 :
12980 66919 : DUMP_VECT_SCOPE ("vectorizable_early_exit");
12981 :
12982 66919 : auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
12983 :
12984 : /* For SLP we don't want to use the type of the operands of the SLP node, when
12985 : vectorizing using SLP slp_node will be the children of the gcond and we
12986 : want to use the type of the direct children which since the gcond is root
12987 : will be the current node, rather than a child node as vect_is_simple_use
12988 : assumes. */
12989 66919 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12990 66919 : if (!vectype)
12991 : return false;
12992 :
12993 66919 : machine_mode mode = TYPE_MODE (vectype);
12994 66919 : int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
12995 :
12996 66919 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
12997 66919 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
12998 66919 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12999 66919 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
13000 :
13001 : /* Now build the new conditional. Pattern gimple_conds get dropped during
13002 : codegen so we must replace the original insn. */
13003 66919 : gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
13004 66919 : gcond *cond_stmt = as_a <gcond *>(orig_stmt);
13005 :
13006 66919 : tree vectype_out = vectype;
13007 66919 : auto bb = gimple_bb (cond_stmt);
13008 66919 : edge exit_true_edge = EDGE_SUCC (bb, 0);
13009 66919 : if (exit_true_edge->flags & EDGE_FALSE_VALUE)
13010 662 : exit_true_edge = EDGE_SUCC (bb, 1);
13011 66919 : gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
13012 :
13013 : /* When vectorizing we assume that if the branch edge is taken that we're
13014 : exiting the loop. This is not however always the case as the compiler will
13015 : rewrite conditions to always be a comparison against 0. To do this it
13016 : sometimes flips the edges. This is fine for scalar, but for vector we
13017 : then have to negate the result of the test, as we're still assuming that if
13018 : you take the branch edge that we found the exit condition. i.e. we need to
13019 : know whether we are generating a `forall` or an `exist` condition. */
13020 133838 : bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
13021 66919 : exit_true_edge->dest);
13022 :
13023 : /* See if we support ADDHN and use that for the reduction. */
13024 66919 : internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
13025 66919 : bool addhn_supported_p
13026 66919 : = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
13027 66919 : tree narrow_type = NULL_TREE;
13028 66919 : if (addhn_supported_p)
13029 : {
13030 : /* Calculate the narrowing type for the result. */
13031 0 : auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
13032 0 : auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
13033 0 : tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
13034 0 : tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
13035 0 : narrow_type = truth_type_for (tmp_type);
13036 :
13037 0 : if (!supports_vector_compare_and_branch (loop_vinfo,
13038 0 : TYPE_MODE (narrow_type)))
13039 : {
13040 0 : if (dump_enabled_p ())
13041 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13042 : "can't use ADDHN reduction because cbranch for "
13043 : "the narrowed type is not supported by the "
13044 : "target.\n");
13045 : addhn_supported_p = false;
13046 : }
13047 : }
13048 :
13049 : /* Analyze only. */
13050 66919 : if (cost_vec)
13051 : {
13052 65330 : if (!addhn_supported_p
13053 65330 : && !supports_vector_compare_and_branch (loop_vinfo, mode))
13054 : {
13055 62701 : if (dump_enabled_p ())
13056 597 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13057 : "can't vectorize early exit because the "
13058 : "target doesn't support flag setting vector "
13059 : "comparisons.\n");
13060 62701 : return false;
13061 : }
13062 :
13063 2629 : if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
13064 : slp_node, cost_vec))
13065 : return false;
13066 :
13067 2629 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
13068 : {
13069 1566 : if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
13070 : OPTIMIZE_FOR_SPEED))
13071 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
13072 : else
13073 1566 : vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
13074 : }
13075 :
13076 2629 : if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
13077 : return false;
13078 :
13079 : return true;
13080 : }
13081 :
13082 : /* Transform. */
13083 :
13084 1589 : tree new_temp = NULL_TREE;
13085 1589 : gimple *new_stmt = NULL;
13086 :
13087 1589 : if (dump_enabled_p ())
13088 406 : dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
13089 :
13090 : /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
13091 : roots and so by the time we get to them we have already codegened the SLP tree
13092 : and so we shouldn't try to do so again. The arguments have already been
13093 : vectorized. It's not very clean to do this here, But the masking code below is
13094 : complex and this keeps it all in one place to ease fixes and backports. Once we
13095 : drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
13096 :
13097 1589 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
13098 1589 : basic_block cond_bb = gimple_bb (stmt);
13099 1589 : gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
13100 :
13101 1589 : auto_vec<tree> stmts;
13102 1589 : stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
13103 :
13104 : /* If we're comparing against a previous forall we need to negate the results
13105 : before we do the final comparison or reduction. */
13106 1589 : if (flipped)
13107 : {
13108 : /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
13109 : if (any(~mask)) by negating the masks and flipping the branches.
13110 :
13111 : 1. For unmasked loops we simply reduce the ~mask.
13112 : 2. For masked loops we reduce (~mask & loop_mask) which is the same as
13113 : doing (mask & loop_mask) ^ loop_mask. */
13114 294 : for (unsigned i = 0; i < stmts.length (); i++)
13115 : {
13116 173 : tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
13117 173 : auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
13118 173 : vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
13119 : &cond_gsi);
13120 173 : stmts[i] = inv_lhs;
13121 : }
13122 :
13123 121 : EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
13124 121 : EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
13125 : }
13126 :
13127 : /* Determine if we need to reduce the final value. */
13128 1589 : if (stmts.length () > 1)
13129 : {
13130 : /* We build the reductions in a way to maintain as much parallelism as
13131 : possible. */
13132 142 : auto_vec<tree> workset (stmts.length ());
13133 :
13134 : /* Mask the statements as we queue them up. Normally we loop over
13135 : vec_num, but since we inspect the exact results of vectorization
13136 : we don't need to and instead can just use the stmts themselves. */
13137 142 : if (masked_loop_p)
13138 0 : for (unsigned i = 0; i < stmts.length (); i++)
13139 : {
13140 0 : tree stmt_mask
13141 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
13142 : vectype, i);
13143 0 : stmt_mask
13144 0 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
13145 0 : stmts[i], &cond_gsi);
13146 0 : workset.quick_push (stmt_mask);
13147 : }
13148 142 : else if (len_loop_p)
13149 0 : for (unsigned i = 0; i < stmts.length (); i++)
13150 : {
13151 0 : tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
13152 : lens, vec_num,
13153 0 : vectype, stmts[i], i, 1);
13154 :
13155 0 : workset.quick_push (len_mask);
13156 : }
13157 : else
13158 142 : workset.splice (stmts);
13159 :
13160 432 : while (workset.length () > 1)
13161 : {
13162 290 : tree arg0 = workset.pop ();
13163 290 : tree arg1 = workset.pop ();
13164 290 : if (addhn_supported_p && workset.length () == 0)
13165 : {
13166 0 : new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
13167 0 : vectype_out = narrow_type;
13168 0 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
13169 0 : gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
13170 0 : gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
13171 : }
13172 : else
13173 : {
13174 290 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
13175 290 : new_stmt
13176 290 : = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
13177 : }
13178 290 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
13179 : &cond_gsi);
13180 290 : workset.quick_insert (0, new_temp);
13181 : }
13182 142 : }
13183 : else
13184 : {
13185 1447 : new_temp = stmts[0];
13186 1447 : if (masked_loop_p)
13187 : {
13188 0 : tree mask
13189 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
13190 0 : new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
13191 : new_temp, &cond_gsi);
13192 : }
13193 1447 : else if (len_loop_p)
13194 0 : new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
13195 : 1, vectype, new_temp, 0, 1);
13196 : }
13197 :
13198 1589 : gcc_assert (new_temp);
13199 :
13200 1589 : tree cst = build_zero_cst (vectype_out);
13201 1589 : gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
13202 1589 : update_stmt (orig_stmt);
13203 :
13204 : /* ??? */
13205 1589 : SLP_TREE_VEC_DEFS (slp_node).truncate (0);
13206 :
13207 1589 : return true;
13208 1589 : }
13209 :
13210 : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13211 : can handle all live statements in the node. Otherwise return true
13212 : if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13213 : VEC_STMT_P is as for vectorizable_live_operation. */
13214 :
13215 : static bool
13216 1307548 : can_vectorize_live_stmts (vec_info *vinfo,
13217 : slp_tree slp_node, slp_instance slp_node_instance,
13218 : bool vec_stmt_p,
13219 : stmt_vector_for_cost *cost_vec)
13220 : {
13221 1307548 : stmt_vec_info slp_stmt_info;
13222 1307548 : unsigned int i;
13223 2756192 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
13224 : {
13225 1448644 : if (slp_stmt_info
13226 1432443 : && STMT_VINFO_LIVE_P (slp_stmt_info)
13227 1585534 : && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13228 : slp_node_instance, i,
13229 : vec_stmt_p, cost_vec))
13230 : return false;
13231 : }
13232 :
13233 : return true;
13234 : }
13235 :
13236 : /* Make sure the statement is vectorizable. */
13237 :
13238 : opt_result
13239 2714562 : vect_analyze_stmt (vec_info *vinfo,
13240 : slp_tree node, slp_instance node_instance,
13241 : stmt_vector_for_cost *cost_vec)
13242 : {
13243 2714562 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
13244 2714562 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
13245 2714562 : enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
13246 2714562 : bool ok;
13247 :
13248 2714562 : if (dump_enabled_p ())
13249 100636 : dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
13250 : stmt_info->stmt);
13251 :
13252 5139772 : if (gimple_has_volatile_ops (stmt_info->stmt))
13253 : {
13254 : /* ??? This shouldn't really happen, volatile stmts should
13255 : not end up in the SLP graph. */
13256 0 : return opt_result::failure_at (stmt_info->stmt,
13257 : "not vectorized:"
13258 : " stmt has volatile operands: %G\n",
13259 : stmt_info->stmt);
13260 : }
13261 :
13262 : /* Skip stmts that do not need to be vectorized. */
13263 2714562 : if (!STMT_VINFO_RELEVANT_P (stmt_info)
13264 0 : && !STMT_VINFO_LIVE_P (stmt_info))
13265 : {
13266 0 : if (dump_enabled_p ())
13267 0 : dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
13268 :
13269 : /* ??? This shouldn't really happen, irrelevant stmts should
13270 : not end up in the SLP graph. */
13271 0 : return opt_result::failure_at (stmt_info->stmt,
13272 : "not vectorized:"
13273 : " irrelevant stmt as SLP node %p "
13274 : "representative.\n",
13275 : (void *)node);
13276 : }
13277 :
13278 2714562 : switch (STMT_VINFO_DEF_TYPE (stmt_info))
13279 : {
13280 : case vect_internal_def:
13281 : case vect_condition_def:
13282 : break;
13283 :
13284 84421 : case vect_reduction_def:
13285 84421 : case vect_nested_cycle:
13286 84421 : gcc_assert (!bb_vinfo
13287 : && (relevance == vect_used_in_outer
13288 : || relevance == vect_used_in_outer_by_reduction
13289 : || relevance == vect_used_by_reduction
13290 : || relevance == vect_unused_in_scope
13291 : || relevance == vect_used_only_live));
13292 : break;
13293 :
13294 322 : case vect_double_reduction_def:
13295 322 : gcc_assert (!bb_vinfo && node);
13296 : break;
13297 :
13298 153228 : case vect_induction_def:
13299 153228 : case vect_first_order_recurrence:
13300 153228 : gcc_assert (!bb_vinfo);
13301 : break;
13302 :
13303 0 : case vect_constant_def:
13304 0 : case vect_external_def:
13305 0 : case vect_unknown_def_type:
13306 0 : default:
13307 0 : gcc_unreachable ();
13308 : }
13309 :
13310 2714562 : tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
13311 2714562 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13312 :
13313 2714562 : if (STMT_VINFO_RELEVANT_P (stmt_info))
13314 : {
13315 2714562 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
13316 2714562 : gcc_assert (SLP_TREE_VECTYPE (node)
13317 : || gimple_code (stmt_info->stmt) == GIMPLE_COND
13318 : || (call && gimple_call_lhs (call) == NULL_TREE));
13319 : }
13320 :
13321 2714562 : ok = true;
13322 2714562 : if (bb_vinfo
13323 1494045 : || (STMT_VINFO_RELEVANT_P (stmt_info)
13324 0 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
13325 : /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13326 : -mveclibabi= takes preference over library functions with
13327 : the simd attribute. */
13328 2714562 : ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
13329 2707678 : || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
13330 : cost_vec)
13331 2707223 : || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
13332 2623036 : || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
13333 2081370 : || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
13334 2011279 : || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
13335 1565431 : || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
13336 739759 : || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
13337 689652 : || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
13338 663243 : || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
13339 520196 : || (bb_vinfo
13340 130266 : && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
13341 3176261 : || (is_a <loop_vec_info> (vinfo)
13342 389930 : && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
13343 : stmt_info, node, cost_vec)
13344 389208 : || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
13345 : stmt_info,
13346 : node, node_instance, cost_vec)
13347 307328 : || vectorizable_induction (as_a <loop_vec_info> (vinfo),
13348 : stmt_info, node, cost_vec)
13349 187579 : || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
13350 : stmt_info, node)
13351 186758 : || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13352 : stmt_info, node, cost_vec)
13353 186497 : || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13354 : stmt_info, NULL, node,
13355 : cost_vec))));
13356 :
13357 2714562 : STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
13358 :
13359 2456296 : if (!ok)
13360 258266 : return opt_result::failure_at (stmt_info->stmt,
13361 : "not vectorized:"
13362 : " relevant stmt not supported: %G",
13363 : stmt_info->stmt);
13364 :
13365 : /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13366 : need extra handling, except for vectorizable reductions. */
13367 2456296 : if (!bb_vinfo
13368 1307548 : && (SLP_TREE_TYPE (node) != lc_phi_info_type
13369 821 : || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
13370 1307548 : && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
13371 3763844 : && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
13372 : node, node_instance,
13373 : false, cost_vec))
13374 0 : return opt_result::failure_at (stmt_info->stmt,
13375 : "not vectorized:"
13376 : " live stmt not supported: %G",
13377 : stmt_info->stmt);
13378 :
13379 2456296 : return opt_result::success ();
13380 : }
13381 :
13382 :
13383 : /* Function vect_transform_stmt.
13384 :
13385 : Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13386 :
13387 : bool
13388 983313 : vect_transform_stmt (vec_info *vinfo,
13389 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
13390 : slp_tree slp_node, slp_instance slp_node_instance)
13391 : {
13392 983313 : bool is_store = false;
13393 983313 : bool done;
13394 :
13395 983313 : gcc_assert (slp_node);
13396 :
13397 983313 : if (stmt_info)
13398 982477 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13399 :
13400 983313 : switch (SLP_TREE_TYPE (slp_node))
13401 : {
13402 23004 : case type_demotion_vec_info_type:
13403 23004 : case type_promotion_vec_info_type:
13404 23004 : case type_conversion_vec_info_type:
13405 23004 : done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
13406 23004 : gcc_assert (done);
13407 : break;
13408 :
13409 16322 : case induc_vec_info_type:
13410 16322 : done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
13411 : stmt_info, slp_node, NULL);
13412 16322 : gcc_assert (done);
13413 : break;
13414 :
13415 8676 : case shift_vec_info_type:
13416 8676 : done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
13417 8676 : gcc_assert (done);
13418 : break;
13419 :
13420 114895 : case op_vec_info_type:
13421 114895 : done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
13422 114895 : gcc_assert (done);
13423 : break;
13424 :
13425 16106 : case assignment_vec_info_type:
13426 16106 : done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
13427 16106 : gcc_assert (done);
13428 : break;
13429 :
13430 168028 : case load_vec_info_type:
13431 168028 : done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
13432 168028 : gcc_assert (done);
13433 : break;
13434 :
13435 552820 : case store_vec_info_type:
13436 552820 : done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
13437 552820 : gcc_assert (done);
13438 : is_store = true;
13439 : break;
13440 :
13441 8599 : case condition_vec_info_type:
13442 8599 : done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
13443 8599 : gcc_assert (done);
13444 : break;
13445 :
13446 12608 : case comparison_vec_info_type:
13447 12608 : done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
13448 12608 : gcc_assert (done);
13449 : break;
13450 :
13451 4193 : case call_vec_info_type:
13452 4193 : done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
13453 4193 : break;
13454 :
13455 358 : case call_simd_clone_vec_info_type:
13456 358 : done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
13457 : slp_node, NULL);
13458 358 : break;
13459 :
13460 2670 : case reduc_vec_info_type:
13461 2670 : done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
13462 : gsi, slp_node);
13463 2670 : gcc_assert (done);
13464 : break;
13465 :
13466 23767 : case cycle_phi_info_type:
13467 23767 : done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
13468 : slp_node, slp_node_instance);
13469 23767 : gcc_assert (done);
13470 : break;
13471 :
13472 530 : case lc_phi_info_type:
13473 530 : done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
13474 : stmt_info, slp_node);
13475 530 : gcc_assert (done);
13476 : break;
13477 :
13478 45 : case recurr_info_type:
13479 45 : done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13480 : stmt_info, slp_node, NULL);
13481 45 : gcc_assert (done);
13482 : break;
13483 :
13484 14307 : case phi_info_type:
13485 14307 : done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
13486 : stmt_info, slp_node, NULL);
13487 14307 : gcc_assert (done);
13488 : break;
13489 :
13490 0 : case loop_exit_ctrl_vec_info_type:
13491 0 : done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13492 : stmt_info, gsi, slp_node, NULL);
13493 0 : gcc_assert (done);
13494 : break;
13495 :
13496 16385 : case permute_info_type:
13497 16385 : done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
13498 16385 : gcc_assert (done);
13499 : break;
13500 :
13501 0 : default:
13502 0 : if (!STMT_VINFO_LIVE_P (stmt_info))
13503 : {
13504 0 : if (dump_enabled_p ())
13505 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13506 : "stmt not supported.\n");
13507 0 : gcc_unreachable ();
13508 : }
13509 983313 : done = true;
13510 : }
13511 :
13512 983313 : if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
13513 430493 : && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
13514 : {
13515 : /* Handle stmts whose DEF is used outside the loop-nest that is
13516 : being vectorized. */
13517 583160 : for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
13518 : {
13519 63107 : stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
13520 63107 : done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13521 : slp_node_instance, lane,
13522 : true, NULL);
13523 63107 : gcc_assert (done);
13524 : }
13525 : }
13526 :
13527 983313 : return is_store;
13528 : }
13529 :
13530 :
13531 : /* Remove a group of stores (for SLP or interleaving), free their
13532 : stmt_vec_info. */
13533 :
13534 : void
13535 0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
13536 : {
13537 0 : stmt_vec_info next_stmt_info = first_stmt_info;
13538 :
13539 0 : while (next_stmt_info)
13540 : {
13541 0 : stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
13542 0 : next_stmt_info = vect_orig_stmt (next_stmt_info);
13543 : /* Free the attached stmt_vec_info and remove the stmt. */
13544 0 : vinfo->remove_stmt (next_stmt_info);
13545 0 : next_stmt_info = tmp;
13546 : }
13547 0 : }
13548 :
13549 : /* If NUNITS is nonzero, return a vector type that contains NUNITS
13550 : elements of type SCALAR_TYPE, or null if the target doesn't support
13551 : such a type.
13552 :
13553 : If NUNITS is zero, return a vector type that contains elements of
13554 : type SCALAR_TYPE, choosing whichever vector size the target prefers.
13555 :
13556 : If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13557 : for this vectorization region and want to "autodetect" the best choice.
13558 : Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13559 : and we want the new type to be interoperable with it. PREVAILING_MODE
13560 : in this case can be a scalar integer mode or a vector mode; when it
13561 : is a vector mode, the function acts like a tree-level version of
13562 : related_vector_mode. */
13563 :
13564 : tree
13565 31900964 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
13566 : tree scalar_type, poly_uint64 nunits)
13567 : {
13568 31900964 : tree orig_scalar_type = scalar_type;
13569 31900964 : scalar_mode inner_mode;
13570 31900964 : machine_mode simd_mode;
13571 31900964 : tree vectype;
13572 :
13573 31900964 : if ((!INTEGRAL_TYPE_P (scalar_type)
13574 10757578 : && !POINTER_TYPE_P (scalar_type)
13575 1828482 : && !SCALAR_FLOAT_TYPE_P (scalar_type))
13576 42140699 : || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
13577 1310721 : && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
13578 521111 : return NULL_TREE;
13579 :
13580 31379853 : unsigned int nbytes = GET_MODE_SIZE (inner_mode);
13581 :
13582 : /* Interoperability between modes requires one to be a constant multiple
13583 : of the other, so that the number of vectors required for each operation
13584 : is a compile-time constant. */
13585 31379853 : if (prevailing_mode != VOIDmode
13586 30225011 : && !constant_multiple_p (nunits * nbytes,
13587 30225011 : GET_MODE_SIZE (prevailing_mode))
13588 32912961 : && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
13589 1533108 : nunits * nbytes))
13590 : return NULL_TREE;
13591 :
13592 : /* For vector types of elements whose mode precision doesn't
13593 : match their types precision we use a element type of mode
13594 : precision. The vectorization routines will have to make sure
13595 : they support the proper result truncation/extension.
13596 : We also make sure to build vector types with INTEGER_TYPE
13597 : component type only. */
13598 31379853 : if (INTEGRAL_TYPE_P (scalar_type)
13599 52523157 : && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
13600 19599271 : || TREE_CODE (scalar_type) != INTEGER_TYPE))
13601 1761306 : scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
13602 1761306 : TYPE_UNSIGNED (scalar_type));
13603 :
13604 : /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13605 : When the component mode passes the above test simply use a type
13606 : corresponding to that mode. The theory is that any use that
13607 : would cause problems with this will disable vectorization anyway. */
13608 29618547 : else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
13609 : && !INTEGRAL_TYPE_P (scalar_type))
13610 8929096 : scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
13611 :
13612 : /* We can't build a vector type of elements with alignment bigger than
13613 : their size. */
13614 20689451 : else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
13615 405664 : scalar_type = lang_hooks.types.type_for_mode (inner_mode,
13616 202832 : TYPE_UNSIGNED (scalar_type));
13617 :
13618 : /* If we felt back to using the mode fail if there was
13619 : no scalar type for it. */
13620 31379853 : if (scalar_type == NULL_TREE)
13621 : return NULL_TREE;
13622 :
13623 : /* If no prevailing mode was supplied, use the mode the target prefers.
13624 : Otherwise lookup a vector mode based on the prevailing mode. */
13625 31379853 : if (prevailing_mode == VOIDmode)
13626 : {
13627 1154842 : gcc_assert (known_eq (nunits, 0U));
13628 1154842 : simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
13629 1154842 : if (SCALAR_INT_MODE_P (simd_mode))
13630 : {
13631 : /* Traditional behavior is not to take the integer mode
13632 : literally, but simply to use it as a way of determining
13633 : the vector size. It is up to mode_for_vector to decide
13634 : what the TYPE_MODE should be.
13635 :
13636 : Note that nunits == 1 is allowed in order to support single
13637 : element vector types. */
13638 59374 : if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
13639 616 : || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13640 29071 : return NULL_TREE;
13641 : }
13642 : }
13643 30225011 : else if (SCALAR_INT_MODE_P (prevailing_mode)
13644 30225011 : || !related_vector_mode (prevailing_mode,
13645 28111843 : inner_mode, nunits).exists (&simd_mode))
13646 : {
13647 : /* Fall back to using mode_for_vector, mostly in the hope of being
13648 : able to use an integer mode. */
13649 2113168 : if (known_eq (nunits, 0U)
13650 4922600 : && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
13651 : return NULL_TREE;
13652 :
13653 153419 : if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13654 143371 : return NULL_TREE;
13655 : }
13656 :
13657 29247662 : vectype = build_vector_type_for_mode (scalar_type, simd_mode);
13658 :
13659 : /* In cases where the mode was chosen by mode_for_vector, check that
13660 : the target actually supports the chosen mode, or that it at least
13661 : allows the vector mode to be replaced by a like-sized integer. */
13662 58495324 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
13663 29257979 : && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
13664 : return NULL_TREE;
13665 :
13666 : /* Re-attach the address-space qualifier if we canonicalized the scalar
13667 : type. */
13668 29239579 : if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
13669 5 : return build_qualified_type
13670 5 : (vectype, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type)));
13671 :
13672 : return vectype;
13673 : }
13674 :
13675 : /* Function get_vectype_for_scalar_type.
13676 :
13677 : Returns the vector type corresponding to SCALAR_TYPE as supported
13678 : by the target. If GROUP_SIZE is nonzero and we're performing BB
13679 : vectorization, make sure that the number of elements in the vector
13680 : is no bigger than GROUP_SIZE. */
13681 :
13682 : tree
13683 27231215 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
13684 : unsigned int group_size)
13685 : {
13686 : /* For BB vectorization, we should always have a group size once we've
13687 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13688 : are tentative requests during things like early data reference
13689 : analysis and pattern recognition. */
13690 27231215 : if (is_a <bb_vec_info> (vinfo))
13691 24354492 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
13692 : else
13693 : group_size = 0;
13694 :
13695 27231215 : tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13696 : scalar_type);
13697 27231215 : if (vectype && vinfo->vector_mode == VOIDmode)
13698 1077829 : vinfo->vector_mode = TYPE_MODE (vectype);
13699 :
13700 : /* Register the natural choice of vector type, before the group size
13701 : has been applied. */
13702 0 : if (vectype)
13703 24741189 : vinfo->used_vector_modes.add (TYPE_MODE (vectype));
13704 :
13705 : /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13706 : try again with an explicit number of elements. */
13707 24741189 : if (vectype
13708 24741189 : && group_size
13709 27231215 : && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
13710 : {
13711 : /* Start with the biggest number of units that fits within
13712 : GROUP_SIZE and halve it until we find a valid vector type.
13713 : Usually either the first attempt will succeed or all will
13714 : fail (in the latter case because GROUP_SIZE is too small
13715 : for the target), but it's possible that a target could have
13716 : a hole between supported vector types.
13717 :
13718 : If GROUP_SIZE is not a power of 2, this has the effect of
13719 : trying the largest power of 2 that fits within the group,
13720 : even though the group is not a multiple of that vector size.
13721 : The BB vectorizer will then try to carve up the group into
13722 : smaller pieces. */
13723 3141495 : unsigned int nunits = 1 << floor_log2 (group_size);
13724 3141495 : do
13725 : {
13726 3141495 : vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13727 3141495 : scalar_type, nunits);
13728 3141495 : nunits /= 2;
13729 : }
13730 3141495 : while (nunits > 1 && !vectype);
13731 : }
13732 :
13733 27231215 : return vectype;
13734 : }
13735 :
13736 : /* Return the vector type corresponding to SCALAR_TYPE as supported
13737 : by the target. NODE, if nonnull, is the SLP tree node that will
13738 : use the returned vector type. */
13739 :
13740 : tree
13741 169723 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
13742 : {
13743 169723 : unsigned int group_size = 0;
13744 169723 : if (node)
13745 169723 : group_size = SLP_TREE_LANES (node);
13746 169723 : return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13747 : }
13748 :
13749 : /* Function get_mask_type_for_scalar_type.
13750 :
13751 : Returns the mask type corresponding to a result of comparison
13752 : of vectors of specified SCALAR_TYPE as supported by target.
13753 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
13754 : make sure that the number of elements in the vector is no bigger
13755 : than GROUP_SIZE. */
13756 :
13757 : tree
13758 1200369 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13759 : unsigned int group_size)
13760 : {
13761 1200369 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13762 :
13763 1200369 : if (!vectype)
13764 : return NULL;
13765 :
13766 1180462 : return truth_type_for (vectype);
13767 : }
13768 :
13769 : /* Function get_mask_type_for_scalar_type.
13770 :
13771 : Returns the mask type corresponding to a result of comparison
13772 : of vectors of specified SCALAR_TYPE as supported by target.
13773 : NODE, if nonnull, is the SLP tree node that will use the returned
13774 : vector type. */
13775 :
13776 : tree
13777 19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13778 : slp_tree node)
13779 : {
13780 19 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
13781 :
13782 19 : if (!vectype)
13783 : return NULL;
13784 :
13785 19 : return truth_type_for (vectype);
13786 : }
13787 :
13788 : /* Function get_same_sized_vectype
13789 :
13790 : Returns a vector type corresponding to SCALAR_TYPE of size
13791 : VECTOR_TYPE if supported by the target. */
13792 :
13793 : tree
13794 160889 : get_same_sized_vectype (tree scalar_type, tree vector_type)
13795 : {
13796 160889 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
13797 0 : return truth_type_for (vector_type);
13798 :
13799 160889 : poly_uint64 nunits;
13800 321778 : if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
13801 321778 : GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
13802 : return NULL_TREE;
13803 :
13804 160889 : return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
13805 160889 : scalar_type, nunits);
13806 : }
13807 :
13808 : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13809 : would not change the chosen vector modes. */
13810 :
13811 : bool
13812 1611346 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
13813 : {
13814 1611346 : for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
13815 3679398 : i != vinfo->used_vector_modes.end (); ++i)
13816 1890878 : if (!VECTOR_MODE_P (*i)
13817 5672634 : || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
13818 856852 : return false;
13819 754494 : return true;
13820 : }
13821 :
13822 : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
13823 : change the chosen vector modes for analysis of a loop. */
13824 :
13825 : bool
13826 390234 : vect_chooses_same_modes_p (machine_mode vector_mode,
13827 : machine_mode alt_vector_mode)
13828 : {
13829 63576 : return (VECTOR_MODE_P (vector_mode)
13830 390234 : && VECTOR_MODE_P (alt_vector_mode)
13831 780468 : && (related_vector_mode (vector_mode,
13832 : GET_MODE_INNER (alt_vector_mode))
13833 390234 : == alt_vector_mode)
13834 416622 : && (related_vector_mode (alt_vector_mode,
13835 : GET_MODE_INNER (vector_mode))
13836 13194 : == vector_mode));
13837 : }
13838 :
13839 : /* Function vect_is_simple_use.
13840 :
13841 : Input:
13842 : VINFO - the vect info of the loop or basic block that is being vectorized.
13843 : OPERAND - operand in the loop or bb.
13844 : Output:
13845 : DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13846 : case OPERAND is an SSA_NAME that is defined in the vectorizable region
13847 : DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13848 : the definition could be anywhere in the function
13849 : DT - the type of definition
13850 :
13851 : Returns whether a stmt with OPERAND can be vectorized.
13852 : For loops, supportable operands are constants, loop invariants, and operands
13853 : that are defined by the current iteration of the loop. Unsupportable
13854 : operands are those that are defined by a previous iteration of the loop (as
13855 : is the case in reduction/induction computations).
13856 : For basic blocks, supportable operands are constants and bb invariants.
13857 : For now, operands defined outside the basic block are not supported. */
13858 :
13859 : bool
13860 42446975 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
13861 : stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
13862 : {
13863 42446975 : if (def_stmt_info_out)
13864 40225585 : *def_stmt_info_out = NULL;
13865 42446975 : if (def_stmt_out)
13866 10008311 : *def_stmt_out = NULL;
13867 42446975 : *dt = vect_unknown_def_type;
13868 :
13869 42446975 : if (dump_enabled_p ())
13870 : {
13871 768004 : dump_printf_loc (MSG_NOTE, vect_location,
13872 : "vect_is_simple_use: operand ");
13873 768004 : if (TREE_CODE (operand) == SSA_NAME
13874 768004 : && !SSA_NAME_IS_DEFAULT_DEF (operand))
13875 704892 : dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
13876 : else
13877 63112 : dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
13878 : }
13879 :
13880 42446975 : if (CONSTANT_CLASS_P (operand))
13881 2890876 : *dt = vect_constant_def;
13882 39556099 : else if (is_gimple_min_invariant (operand))
13883 339956 : *dt = vect_external_def;
13884 39216143 : else if (TREE_CODE (operand) != SSA_NAME)
13885 982 : *dt = vect_unknown_def_type;
13886 39215161 : else if (SSA_NAME_IS_DEFAULT_DEF (operand))
13887 516431 : *dt = vect_external_def;
13888 : else
13889 : {
13890 38698730 : gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
13891 38698730 : stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
13892 38698730 : if (!stmt_vinfo)
13893 851461 : *dt = vect_external_def;
13894 : else
13895 : {
13896 37847269 : stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
13897 37847269 : def_stmt = stmt_vinfo->stmt;
13898 37847269 : *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
13899 37847269 : if (def_stmt_info_out)
13900 35634924 : *def_stmt_info_out = stmt_vinfo;
13901 : }
13902 38698730 : if (def_stmt_out)
13903 9797976 : *def_stmt_out = def_stmt;
13904 : }
13905 :
13906 42446975 : if (dump_enabled_p ())
13907 : {
13908 768004 : dump_printf (MSG_NOTE, ", type of def: ");
13909 768004 : switch (*dt)
13910 : {
13911 0 : case vect_uninitialized_def:
13912 0 : dump_printf (MSG_NOTE, "uninitialized\n");
13913 0 : break;
13914 52383 : case vect_constant_def:
13915 52383 : dump_printf (MSG_NOTE, "constant\n");
13916 52383 : break;
13917 26300 : case vect_external_def:
13918 26300 : dump_printf (MSG_NOTE, "external\n");
13919 26300 : break;
13920 549484 : case vect_internal_def:
13921 549484 : dump_printf (MSG_NOTE, "internal\n");
13922 549484 : break;
13923 108307 : case vect_induction_def:
13924 108307 : dump_printf (MSG_NOTE, "induction\n");
13925 108307 : break;
13926 28165 : case vect_reduction_def:
13927 28165 : dump_printf (MSG_NOTE, "reduction\n");
13928 28165 : break;
13929 482 : case vect_double_reduction_def:
13930 482 : dump_printf (MSG_NOTE, "double reduction\n");
13931 482 : break;
13932 2173 : case vect_nested_cycle:
13933 2173 : dump_printf (MSG_NOTE, "nested cycle\n");
13934 2173 : break;
13935 276 : case vect_first_order_recurrence:
13936 276 : dump_printf (MSG_NOTE, "first order recurrence\n");
13937 276 : break;
13938 0 : case vect_condition_def:
13939 0 : dump_printf (MSG_NOTE, "control flow\n");
13940 0 : break;
13941 434 : case vect_unknown_def_type:
13942 434 : dump_printf (MSG_NOTE, "unknown\n");
13943 434 : break;
13944 : }
13945 : }
13946 :
13947 42446975 : if (*dt == vect_unknown_def_type)
13948 : {
13949 58278 : if (dump_enabled_p ())
13950 434 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13951 : "Unsupported pattern.\n");
13952 58278 : return false;
13953 : }
13954 :
13955 : return true;
13956 : }
13957 :
13958 : /* Function vect_is_simple_use.
13959 :
13960 : Same as vect_is_simple_use but determines the operand by operand
13961 : position OPERAND from either STMT or SLP_NODE, filling in *OP
13962 : and *SLP_DEF (when SLP_NODE is not NULL). */
13963 :
13964 : bool
13965 3900256 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13966 : unsigned operand, tree *op, slp_tree *slp_def,
13967 : enum vect_def_type *dt,
13968 : tree *vectype, stmt_vec_info *def_stmt_info_out)
13969 : {
13970 3900256 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
13971 3900256 : *slp_def = child;
13972 3900256 : *vectype = SLP_TREE_VECTYPE (child);
13973 3900256 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
13974 : {
13975 : /* ??? VEC_PERM nodes might be intermediate and their lane value
13976 : have no representative (nor do we build a VEC_PERM stmt for
13977 : the actual operation). Note for two-operator nodes we set
13978 : a representative but leave scalar stmts empty as we'd only
13979 : have one for a subset of lanes. Ideally no caller would
13980 : require *op for internal defs. */
13981 2152475 : if (SLP_TREE_REPRESENTATIVE (child))
13982 : {
13983 2151698 : *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
13984 2151698 : return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
13985 : }
13986 : else
13987 : {
13988 777 : gcc_assert (SLP_TREE_PERMUTE_P (child));
13989 777 : *op = error_mark_node;
13990 777 : *dt = vect_internal_def;
13991 777 : if (def_stmt_info_out)
13992 0 : *def_stmt_info_out = NULL;
13993 777 : return true;
13994 : }
13995 : }
13996 : else
13997 : {
13998 1747781 : if (def_stmt_info_out)
13999 55982 : *def_stmt_info_out = NULL;
14000 1747781 : *op = SLP_TREE_SCALAR_OPS (child)[0];
14001 1747781 : *dt = SLP_TREE_DEF_TYPE (child);
14002 1747781 : return true;
14003 : }
14004 : }
14005 :
14006 : /* If OP is not NULL and is external or constant update its vector
14007 : type with VECTYPE. Returns true if successful or false if not,
14008 : for example when conflicting vector types are present. */
14009 :
14010 : bool
14011 3545606 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
14012 : {
14013 3545606 : if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
14014 : return true;
14015 1174501 : if (SLP_TREE_VECTYPE (op))
14016 103897 : return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
14017 : /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
14018 : should be handled by patters. Allow vect_constant_def for now
14019 : as well as the trivial single-lane uniform vect_external_def case
14020 : both of which we code-generate reasonably. */
14021 1070604 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
14022 1663 : && SLP_TREE_DEF_TYPE (op) == vect_external_def
14023 1071813 : && SLP_TREE_LANES (op) > 1)
14024 : return false;
14025 1070451 : SLP_TREE_VECTYPE (op) = vectype;
14026 1070451 : return true;
14027 : }
14028 :
14029 : /* Function supportable_widening_operation
14030 :
14031 : Check whether an operation represented by the code CODE is a
14032 : widening operation that is supported by the target platform in
14033 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14034 : producing a result of type VECTYPE_OUT).
14035 :
14036 : Widening operations we currently support are NOP (CONVERT), FLOAT,
14037 : FIX_TRUNC and WIDEN_MULT. This function checks if these operations
14038 : are supported by the target platform either directly (via vector
14039 : tree-codes), or via target builtins.
14040 :
14041 : When EVENODD_OK then also lane-swizzling operations are considered.
14042 :
14043 : Output:
14044 : - CODE1 and CODE2 are codes of vector operations to be used when
14045 : vectorizing the operation, if available.
14046 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14047 : case of multi-step conversion (like char->short->int - in that case
14048 : MULTI_STEP_CVT will be 1).
14049 : - INTERM_TYPES contains the intermediate type required to perform the
14050 : widening operation (short in the above example). */
14051 :
14052 : bool
14053 498899 : supportable_widening_operation (code_helper code,
14054 : tree vectype_out, tree vectype_in,
14055 : bool evenodd_ok,
14056 : code_helper *code1,
14057 : code_helper *code2,
14058 : int *multi_step_cvt,
14059 : vec<tree> *interm_types)
14060 : {
14061 498899 : machine_mode vec_mode;
14062 498899 : enum insn_code icode1, icode2;
14063 498899 : optab optab1 = unknown_optab, optab2 = unknown_optab;
14064 498899 : tree vectype = vectype_in;
14065 498899 : tree wide_vectype = vectype_out;
14066 498899 : tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
14067 498899 : int i;
14068 498899 : tree prev_type, intermediate_type;
14069 498899 : machine_mode intermediate_mode, prev_mode;
14070 498899 : optab optab3, optab4;
14071 :
14072 498899 : *multi_step_cvt = 0;
14073 :
14074 498899 : switch (code.safe_as_tree_code ())
14075 : {
14076 : case MAX_TREE_CODES:
14077 : /* Don't set c1 and c2 if code is not a tree_code. */
14078 : break;
14079 :
14080 193244 : case WIDEN_MULT_EXPR:
14081 : /* The result of a vectorized widening operation usually requires
14082 : two vectors (because the widened results do not fit into one vector).
14083 : The generated vector results would normally be expected to be
14084 : generated in the same order as in the original scalar computation,
14085 : i.e. if 8 results are generated in each vector iteration, they are
14086 : to be organized as follows:
14087 : vect1: [res1,res2,res3,res4],
14088 : vect2: [res5,res6,res7,res8].
14089 :
14090 : However, in the special case that the result of the widening
14091 : operation is used in a reduction computation only, the order doesn't
14092 : matter (because when vectorizing a reduction we change the order of
14093 : the computation). Some targets can take advantage of this and
14094 : generate more efficient code. For example, targets like Altivec,
14095 : that support widen_mult using a sequence of {mult_even,mult_odd}
14096 : generate the following vectors:
14097 : vect1: [res1,res3,res5,res7],
14098 : vect2: [res2,res4,res6,res8].
14099 :
14100 : When vectorizing outer-loops, we execute the inner-loop sequentially
14101 : (each vectorized inner-loop iteration contributes to VF outer-loop
14102 : iterations in parallel). We therefore don't allow to change the
14103 : order of the computation in the inner-loop during outer-loop
14104 : vectorization. */
14105 : /* TODO: Another case in which order doesn't *really* matter is when we
14106 : widen and then contract again, e.g. (short)((int)x * y >> 8).
14107 : Normally, pack_trunc performs an even/odd permute, whereas the
14108 : repack from an even/odd expansion would be an interleave, which
14109 : would be significantly simpler for e.g. AVX2. */
14110 : /* In any case, in order to avoid duplicating the code below, recurse
14111 : on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
14112 : are properly set up for the caller. If we fail, we'll continue with
14113 : a VEC_WIDEN_MULT_LO/HI_EXPR check. */
14114 193244 : if (evenodd_ok
14115 193244 : && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
14116 : vectype_out, vectype_in,
14117 : evenodd_ok, code1,
14118 : code2, multi_step_cvt,
14119 : interm_types))
14120 101838 : return true;
14121 : c1 = VEC_WIDEN_MULT_LO_EXPR;
14122 : c2 = VEC_WIDEN_MULT_HI_EXPR;
14123 : break;
14124 :
14125 : case DOT_PROD_EXPR:
14126 397061 : c1 = DOT_PROD_EXPR;
14127 397061 : c2 = DOT_PROD_EXPR;
14128 : break;
14129 :
14130 0 : case SAD_EXPR:
14131 0 : c1 = SAD_EXPR;
14132 0 : c2 = SAD_EXPR;
14133 0 : break;
14134 :
14135 191276 : case VEC_WIDEN_MULT_EVEN_EXPR:
14136 : /* Support the recursion induced just above. */
14137 191276 : c1 = VEC_WIDEN_MULT_EVEN_EXPR;
14138 191276 : c2 = VEC_WIDEN_MULT_ODD_EXPR;
14139 191276 : break;
14140 :
14141 9695 : case WIDEN_LSHIFT_EXPR:
14142 9695 : c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
14143 9695 : c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
14144 9695 : break;
14145 :
14146 41200 : CASE_CONVERT:
14147 41200 : c1 = VEC_UNPACK_LO_EXPR;
14148 41200 : c2 = VEC_UNPACK_HI_EXPR;
14149 41200 : break;
14150 :
14151 9215 : case FLOAT_EXPR:
14152 9215 : c1 = VEC_UNPACK_FLOAT_LO_EXPR;
14153 9215 : c2 = VEC_UNPACK_FLOAT_HI_EXPR;
14154 9215 : break;
14155 :
14156 119 : case FIX_TRUNC_EXPR:
14157 119 : c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
14158 119 : c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
14159 119 : break;
14160 :
14161 0 : default:
14162 0 : gcc_unreachable ();
14163 : }
14164 :
14165 397061 : if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
14166 : std::swap (c1, c2);
14167 :
14168 397061 : if (code == FIX_TRUNC_EXPR)
14169 : {
14170 : /* The signedness is determined from output operand. */
14171 119 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14172 119 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14173 : }
14174 706847 : else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
14175 41200 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14176 7943 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14177 7943 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14178 343256 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14179 : {
14180 : /* If the input and result modes are the same, a different optab
14181 : is needed where we pass in the number of units in vectype. */
14182 : optab1 = vec_unpacks_sbool_lo_optab;
14183 : optab2 = vec_unpacks_sbool_hi_optab;
14184 : }
14185 :
14186 397061 : vec_mode = TYPE_MODE (vectype);
14187 397061 : if (widening_fn_p (code))
14188 : {
14189 : /* If this is an internal fn then we must check whether the target
14190 : supports either a low-high split or an even-odd split. */
14191 54150 : internal_fn ifn = as_internal_fn ((combined_fn) code);
14192 :
14193 54150 : internal_fn lo, hi, even, odd;
14194 54150 : lookup_hilo_internal_fn (ifn, &lo, &hi);
14195 54150 : if (BYTES_BIG_ENDIAN)
14196 : std::swap (lo, hi);
14197 54150 : *code1 = as_combined_fn (lo);
14198 54150 : *code2 = as_combined_fn (hi);
14199 54150 : optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
14200 54150 : optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
14201 :
14202 : /* If we don't support low-high, then check for even-odd. */
14203 54150 : if (!optab1
14204 54150 : || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14205 0 : || !optab2
14206 54150 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14207 : {
14208 54150 : lookup_evenodd_internal_fn (ifn, &even, &odd);
14209 54150 : *code1 = as_combined_fn (even);
14210 54150 : *code2 = as_combined_fn (odd);
14211 54150 : optab1 = direct_internal_fn_optab (even, {vectype, vectype});
14212 54150 : optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
14213 : }
14214 : }
14215 342911 : else if (code.is_tree_code ())
14216 : {
14217 342911 : if (code == FIX_TRUNC_EXPR)
14218 : {
14219 : /* The signedness is determined from output operand. */
14220 119 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14221 119 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14222 : }
14223 342792 : else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
14224 41200 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14225 7943 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14226 7943 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14227 343256 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14228 : {
14229 : /* If the input and result modes are the same, a different optab
14230 : is needed where we pass in the number of units in vectype. */
14231 : optab1 = vec_unpacks_sbool_lo_optab;
14232 : optab2 = vec_unpacks_sbool_hi_optab;
14233 : }
14234 : else
14235 : {
14236 342328 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14237 342328 : optab2 = optab_for_tree_code (c2, vectype, optab_default);
14238 : }
14239 342911 : *code1 = c1;
14240 342911 : *code2 = c2;
14241 : }
14242 :
14243 397061 : if (!optab1 || !optab2)
14244 : return false;
14245 :
14246 397061 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14247 397061 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14248 235914 : return false;
14249 :
14250 :
14251 161147 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14252 161147 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14253 : {
14254 149625 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14255 : return true;
14256 : /* For scalar masks we may have different boolean
14257 : vector types having the same QImode. Thus we
14258 : add additional check for elements number. */
14259 4230 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
14260 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14261 : return true;
14262 : }
14263 :
14264 : /* Check if it's a multi-step conversion that can be done using intermediate
14265 : types. */
14266 :
14267 11727 : prev_type = vectype;
14268 11727 : prev_mode = vec_mode;
14269 :
14270 247926 : if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
14271 : return false;
14272 :
14273 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14274 : intermediate steps in promotion sequence. We try
14275 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14276 : not. */
14277 11675 : interm_types->create (MAX_INTERM_CVT_STEPS);
14278 12921 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14279 : {
14280 12921 : intermediate_mode = insn_data[icode1].operand[0].mode;
14281 12921 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14282 4577 : intermediate_type
14283 4577 : = vect_halve_mask_nunits (prev_type, intermediate_mode);
14284 8344 : else if (VECTOR_MODE_P (intermediate_mode))
14285 : {
14286 8344 : tree intermediate_element_type
14287 8344 : = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
14288 8344 : TYPE_UNSIGNED (prev_type));
14289 8344 : intermediate_type
14290 8344 : = build_vector_type_for_mode (intermediate_element_type,
14291 : intermediate_mode);
14292 8344 : }
14293 : else
14294 0 : intermediate_type
14295 0 : = lang_hooks.types.type_for_mode (intermediate_mode,
14296 0 : TYPE_UNSIGNED (prev_type));
14297 :
14298 12921 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14299 4577 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14300 4577 : && intermediate_mode == TYPE_MODE (wide_vectype)
14301 13190 : && SCALAR_INT_MODE_P (intermediate_mode))
14302 : {
14303 : /* If the input and result modes are the same, a different optab
14304 : is needed where we pass in the number of units in vectype. */
14305 : optab3 = vec_unpacks_sbool_lo_optab;
14306 : optab4 = vec_unpacks_sbool_hi_optab;
14307 : }
14308 : else
14309 : {
14310 12652 : optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
14311 12652 : optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
14312 : }
14313 :
14314 12921 : if (!optab3 || !optab4
14315 12921 : || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
14316 12889 : || insn_data[icode1].operand[0].mode != intermediate_mode
14317 12889 : || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
14318 12889 : || insn_data[icode2].operand[0].mode != intermediate_mode
14319 12889 : || ((icode1 = optab_handler (optab3, intermediate_mode))
14320 : == CODE_FOR_nothing)
14321 25557 : || ((icode2 = optab_handler (optab4, intermediate_mode))
14322 : == CODE_FOR_nothing))
14323 : break;
14324 :
14325 12636 : interm_types->quick_push (intermediate_type);
14326 12636 : (*multi_step_cvt)++;
14327 :
14328 12636 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14329 12636 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14330 : {
14331 11454 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14332 : return true;
14333 3711 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
14334 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14335 : return true;
14336 : }
14337 :
14338 1246 : prev_type = intermediate_type;
14339 1246 : prev_mode = intermediate_mode;
14340 : }
14341 :
14342 285 : interm_types->release ();
14343 285 : return false;
14344 : }
14345 :
14346 :
14347 : /* Function supportable_narrowing_operation
14348 :
14349 : Check whether an operation represented by the code CODE is a
14350 : narrowing operation that is supported by the target platform in
14351 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14352 : and producing a result of type VECTYPE_OUT).
14353 :
14354 : Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14355 : and FLOAT. This function checks if these operations are supported by
14356 : the target platform directly via vector tree-codes.
14357 :
14358 : Output:
14359 : - CODE1 is the code of a vector operation to be used when
14360 : vectorizing the operation, if available.
14361 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14362 : case of multi-step conversion (like int->short->char - in that case
14363 : MULTI_STEP_CVT will be 1).
14364 : - INTERM_TYPES contains the intermediate type required to perform the
14365 : narrowing operation (short in the above example). */
14366 :
14367 : bool
14368 42569 : supportable_narrowing_operation (code_helper code,
14369 : tree vectype_out, tree vectype_in,
14370 : code_helper *code1, int *multi_step_cvt,
14371 : vec<tree> *interm_types)
14372 : {
14373 42569 : machine_mode vec_mode;
14374 42569 : enum insn_code icode1;
14375 42569 : optab optab1, interm_optab;
14376 42569 : tree vectype = vectype_in;
14377 42569 : tree narrow_vectype = vectype_out;
14378 42569 : enum tree_code c1;
14379 42569 : tree intermediate_type, prev_type;
14380 42569 : machine_mode intermediate_mode, prev_mode;
14381 42569 : int i;
14382 42569 : unsigned HOST_WIDE_INT n_elts;
14383 42569 : bool uns;
14384 :
14385 42569 : if (!code.is_tree_code ())
14386 : return false;
14387 :
14388 42569 : *multi_step_cvt = 0;
14389 42569 : switch ((tree_code) code)
14390 : {
14391 41719 : CASE_CONVERT:
14392 41719 : c1 = VEC_PACK_TRUNC_EXPR;
14393 41719 : if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
14394 11562 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14395 11562 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
14396 5262 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
14397 46981 : && n_elts < BITS_PER_UNIT)
14398 : optab1 = vec_pack_sbool_trunc_optab;
14399 : else
14400 39234 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14401 : break;
14402 :
14403 561 : case FIX_TRUNC_EXPR:
14404 561 : c1 = VEC_PACK_FIX_TRUNC_EXPR;
14405 : /* The signedness is determined from output operand. */
14406 561 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14407 561 : break;
14408 :
14409 289 : case FLOAT_EXPR:
14410 289 : c1 = VEC_PACK_FLOAT_EXPR;
14411 289 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14412 289 : break;
14413 :
14414 0 : default:
14415 0 : gcc_unreachable ();
14416 : }
14417 :
14418 42569 : if (!optab1)
14419 : return false;
14420 :
14421 42569 : vec_mode = TYPE_MODE (vectype);
14422 42569 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
14423 : return false;
14424 :
14425 37930 : *code1 = c1;
14426 :
14427 37930 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14428 : {
14429 23638 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14430 : return true;
14431 : /* For scalar masks we may have different boolean
14432 : vector types having the same QImode. Thus we
14433 : add additional check for elements number. */
14434 5799 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
14435 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14436 : return true;
14437 : }
14438 :
14439 14443 : if (code == FLOAT_EXPR)
14440 : return false;
14441 :
14442 : /* Check if it's a multi-step conversion that can be done using intermediate
14443 : types. */
14444 14443 : prev_mode = vec_mode;
14445 14443 : prev_type = vectype;
14446 14443 : if (code == FIX_TRUNC_EXPR)
14447 94 : uns = TYPE_UNSIGNED (vectype_out);
14448 : else
14449 14349 : uns = TYPE_UNSIGNED (vectype);
14450 :
14451 : /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14452 : conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14453 : costly than signed. */
14454 14443 : if (code == FIX_TRUNC_EXPR && uns)
14455 : {
14456 28 : enum insn_code icode2;
14457 :
14458 28 : intermediate_type
14459 28 : = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
14460 28 : interm_optab
14461 28 : = optab_for_tree_code (c1, intermediate_type, optab_default);
14462 28 : if (interm_optab != unknown_optab
14463 28 : && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
14464 28 : && insn_data[icode1].operand[0].mode
14465 28 : == insn_data[icode2].operand[0].mode)
14466 : {
14467 : uns = false;
14468 : optab1 = interm_optab;
14469 : icode1 = icode2;
14470 : }
14471 : }
14472 :
14473 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14474 : intermediate steps in promotion sequence. We try
14475 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14476 14443 : interm_types->create (MAX_INTERM_CVT_STEPS);
14477 30970 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14478 : {
14479 16527 : intermediate_mode = insn_data[icode1].operand[0].mode;
14480 16527 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14481 7004 : intermediate_type
14482 7004 : = vect_double_mask_nunits (prev_type, intermediate_mode);
14483 : else
14484 9523 : intermediate_type
14485 9523 : = lang_hooks.types.type_for_mode (intermediate_mode, uns);
14486 16527 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14487 7004 : && VECTOR_BOOLEAN_TYPE_P (prev_type)
14488 7004 : && SCALAR_INT_MODE_P (prev_mode)
14489 3134 : && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
14490 19661 : && n_elts < BITS_PER_UNIT)
14491 : interm_optab = vec_pack_sbool_trunc_optab;
14492 : else
14493 16173 : interm_optab
14494 16173 : = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
14495 : optab_default);
14496 354 : if (!interm_optab
14497 16527 : || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
14498 16527 : || insn_data[icode1].operand[0].mode != intermediate_mode
14499 32700 : || ((icode1 = optab_handler (interm_optab, intermediate_mode))
14500 : == CODE_FOR_nothing))
14501 : break;
14502 :
14503 15612 : interm_types->quick_push (intermediate_type);
14504 15612 : (*multi_step_cvt)++;
14505 :
14506 15612 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14507 : {
14508 13528 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14509 : return true;
14510 4930 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
14511 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14512 : return true;
14513 : }
14514 :
14515 2084 : prev_mode = intermediate_mode;
14516 2084 : prev_type = intermediate_type;
14517 2084 : optab1 = interm_optab;
14518 : }
14519 :
14520 915 : interm_types->release ();
14521 915 : return false;
14522 : }
14523 :
14524 : /* Function supportable_indirect_convert_operation
14525 :
14526 : Check whether an operation represented by the code CODE is single or multi
14527 : operations that are supported by the target platform in
14528 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14529 : producing a result of type VECTYPE_OUT).
14530 :
14531 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14532 : This function checks if these operations are supported
14533 : by the target platform directly (via vector tree-codes).
14534 :
14535 : Output:
14536 : - converts contains some pairs to perform the convert operation,
14537 : the pair's first is the intermediate type, and its second is the code of
14538 : a vector operation to be used when converting the operation from the
14539 : previous type to the intermediate type. */
14540 : bool
14541 87578 : supportable_indirect_convert_operation (code_helper code,
14542 : tree vectype_out,
14543 : tree vectype_in,
14544 : vec<std::pair<tree, tree_code> > &converts,
14545 : tree op0, slp_tree slp_op0)
14546 : {
14547 87578 : bool found_mode = false;
14548 87578 : scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
14549 87578 : scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
14550 87578 : tree_code tc1, tc2, code1, code2;
14551 :
14552 87578 : tree cvt_type = NULL_TREE;
14553 87578 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
14554 :
14555 87578 : if (supportable_convert_operation ((tree_code) code,
14556 : vectype_out,
14557 : vectype_in,
14558 : &tc1))
14559 : {
14560 20011 : converts.safe_push (std::make_pair (vectype_out, tc1));
14561 20011 : return true;
14562 : }
14563 :
14564 : /* For conversions between float and integer types try whether
14565 : we can use intermediate signed integer types to support the
14566 : conversion. */
14567 135134 : if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
14568 67567 : && (code == FLOAT_EXPR
14569 3429 : || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
14570 : {
14571 564 : bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
14572 282 : bool float_expr_p = code == FLOAT_EXPR;
14573 282 : unsigned short target_size;
14574 282 : scalar_mode intermediate_mode;
14575 282 : if (demotion)
14576 : {
14577 90 : intermediate_mode = lhs_mode;
14578 90 : target_size = GET_MODE_SIZE (rhs_mode);
14579 : }
14580 : else
14581 : {
14582 192 : target_size = GET_MODE_SIZE (lhs_mode);
14583 192 : if (!int_mode_for_size
14584 192 : (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
14585 142 : return false;
14586 : }
14587 282 : code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
14588 : code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
14589 282 : opt_scalar_mode mode_iter;
14590 507 : FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
14591 : {
14592 507 : intermediate_mode = mode_iter.require ();
14593 :
14594 1014 : if (GET_MODE_SIZE (intermediate_mode) > target_size)
14595 : break;
14596 :
14597 411 : scalar_mode cvt_mode;
14598 411 : if (!int_mode_for_size
14599 411 : (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
14600 : break;
14601 :
14602 381 : cvt_type = build_nonstandard_integer_type
14603 381 : (GET_MODE_BITSIZE (cvt_mode), 0);
14604 :
14605 : /* Check if the intermediate type can hold OP0's range.
14606 : When converting from float to integer this is not necessary
14607 : because values that do not fit the (smaller) target type are
14608 : unspecified anyway. */
14609 381 : if (demotion && float_expr_p)
14610 : {
14611 14 : wide_int op_min_value, op_max_value;
14612 : /* For vector form, it looks like op0 doesn't have RANGE_INFO.
14613 : In the future, if it is supported, changes may need to be made
14614 : to this part, such as checking the RANGE of each element
14615 : in the vector. */
14616 14 : if (slp_op0)
14617 : {
14618 4 : tree def;
14619 : /* ??? Merge ranges in case of more than one lane. */
14620 4 : if (SLP_TREE_LANES (slp_op0) != 1
14621 0 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
14622 4 : || !vect_get_range_info (def,
14623 : &op_min_value, &op_max_value))
14624 : break;
14625 : }
14626 10 : else if (!op0
14627 0 : || TREE_CODE (op0) != SSA_NAME
14628 0 : || !SSA_NAME_RANGE_INFO (op0)
14629 10 : || !vect_get_range_info (op0, &op_min_value,
14630 : &op_max_value))
14631 : break;
14632 :
14633 0 : if (cvt_type == NULL_TREE
14634 0 : || (wi::min_precision (op_max_value, SIGNED)
14635 0 : > TYPE_PRECISION (cvt_type))
14636 0 : || (wi::min_precision (op_min_value, SIGNED)
14637 0 : > TYPE_PRECISION (cvt_type)))
14638 0 : continue;
14639 14 : }
14640 :
14641 367 : cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
14642 : cvt_type,
14643 : nelts);
14644 : /* This should only happened for SLP as long as loop vectorizer
14645 : only supports same-sized vector. */
14646 592 : if (cvt_type == NULL_TREE
14647 509 : || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
14648 367 : || !supportable_convert_operation ((tree_code) code1,
14649 : vectype_out,
14650 : cvt_type, &tc1)
14651 605 : || !supportable_convert_operation ((tree_code) code2,
14652 : cvt_type,
14653 : vectype_in, &tc2))
14654 225 : continue;
14655 :
14656 : found_mode = true;
14657 : break;
14658 : }
14659 :
14660 282 : if (found_mode)
14661 : {
14662 142 : converts.safe_push (std::make_pair (cvt_type, tc2));
14663 142 : if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
14664 142 : converts.safe_push (std::make_pair (vectype_out, tc1));
14665 142 : return true;
14666 : }
14667 : }
14668 : return false;
14669 : }
14670 :
14671 : /* Generate and return a vector mask of MASK_TYPE such that
14672 : mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14673 : Add the statements to SEQ. */
14674 :
14675 : tree
14676 0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
14677 : tree end_index, const char *name)
14678 : {
14679 0 : tree cmp_type = TREE_TYPE (start_index);
14680 0 : gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
14681 : cmp_type, mask_type,
14682 : OPTIMIZE_FOR_SPEED));
14683 0 : gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
14684 : start_index, end_index,
14685 : build_zero_cst (mask_type));
14686 0 : tree tmp;
14687 0 : if (name)
14688 0 : tmp = make_temp_ssa_name (mask_type, NULL, name);
14689 : else
14690 0 : tmp = make_ssa_name (mask_type);
14691 0 : gimple_call_set_lhs (call, tmp);
14692 0 : gimple_seq_add_stmt (seq, call);
14693 0 : return tmp;
14694 : }
14695 :
14696 : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14697 : J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14698 :
14699 : tree
14700 0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
14701 : tree end_index)
14702 : {
14703 0 : tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
14704 0 : return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
14705 : }
14706 :
14707 : /* Try to compute the vector types required to vectorize STMT_INFO,
14708 : returning true on success and false if vectorization isn't possible.
14709 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
14710 : take sure that the number of elements in the vectors is no bigger
14711 : than GROUP_SIZE.
14712 :
14713 : On success:
14714 :
14715 : - Set *STMT_VECTYPE_OUT to:
14716 : - NULL_TREE if the statement doesn't need to be vectorized;
14717 : - the equivalent of STMT_VINFO_VECTYPE otherwise.
14718 :
14719 : - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14720 : number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14721 : statement does not help to determine the overall number of units. */
14722 :
14723 : opt_result
14724 5894532 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
14725 : tree *stmt_vectype_out,
14726 : tree *nunits_vectype_out,
14727 : unsigned int group_size)
14728 : {
14729 5894532 : gimple *stmt = stmt_info->stmt;
14730 :
14731 : /* For BB vectorization, we should always have a group size once we've
14732 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14733 : are tentative requests during things like early data reference
14734 : analysis and pattern recognition. */
14735 5894532 : if (is_a <bb_vec_info> (vinfo))
14736 4666632 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
14737 : else
14738 : group_size = 0;
14739 :
14740 5894532 : *stmt_vectype_out = NULL_TREE;
14741 5894532 : *nunits_vectype_out = NULL_TREE;
14742 :
14743 5894532 : if (gimple_get_lhs (stmt) == NULL_TREE
14744 : /* Allow vector conditionals through here. */
14745 2679 : && !is_a <gcond *> (stmt)
14746 : /* MASK_STORE and friends have no lhs, but are ok. */
14747 5899870 : && !(is_gimple_call (stmt)
14748 2679 : && gimple_call_internal_p (stmt)
14749 2659 : && internal_store_fn_p (gimple_call_internal_fn (stmt))))
14750 : {
14751 20 : if (is_a <gcall *> (stmt))
14752 : {
14753 : /* Ignore calls with no lhs. These must be calls to
14754 : #pragma omp simd functions, and what vectorization factor
14755 : it really needs can't be determined until
14756 : vectorizable_simd_clone_call. */
14757 20 : if (dump_enabled_p ())
14758 18 : dump_printf_loc (MSG_NOTE, vect_location,
14759 : "defer to SIMD clone analysis.\n");
14760 20 : return opt_result::success ();
14761 : }
14762 :
14763 0 : return opt_result::failure_at (stmt,
14764 : "not vectorized: irregular stmt: %G", stmt);
14765 : }
14766 :
14767 5894512 : tree vectype;
14768 5894512 : tree scalar_type = NULL_TREE;
14769 5894512 : if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
14770 : {
14771 1607414 : vectype = STMT_VINFO_VECTYPE (stmt_info);
14772 1607414 : if (dump_enabled_p ())
14773 79499 : dump_printf_loc (MSG_NOTE, vect_location,
14774 : "precomputed vectype: %T\n", vectype);
14775 : }
14776 4287098 : else if (vect_use_mask_type_p (stmt_info))
14777 : {
14778 220890 : unsigned int precision = stmt_info->mask_precision;
14779 220890 : scalar_type = build_nonstandard_integer_type (precision, 1);
14780 220890 : vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
14781 220890 : if (!vectype)
14782 0 : return opt_result::failure_at (stmt, "not vectorized: unsupported"
14783 : " data-type %T\n", scalar_type);
14784 220890 : if (dump_enabled_p ())
14785 4752 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14786 : }
14787 : else
14788 : {
14789 : /* If we got here with a gcond it means that the target had no available vector
14790 : mode for the scalar type. We can't vectorize so abort. */
14791 4066208 : if (is_a <gcond *> (stmt))
14792 0 : return opt_result::failure_at (stmt,
14793 : "not vectorized:"
14794 : " unsupported data-type for gcond %T\n",
14795 : scalar_type);
14796 :
14797 4066208 : if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
14798 1489299 : scalar_type = TREE_TYPE (DR_REF (dr));
14799 : else
14800 2576909 : scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
14801 :
14802 4066208 : if (dump_enabled_p ())
14803 : {
14804 62337 : if (group_size)
14805 7749 : dump_printf_loc (MSG_NOTE, vect_location,
14806 : "get vectype for scalar type (group size %d):"
14807 : " %T\n", group_size, scalar_type);
14808 : else
14809 54588 : dump_printf_loc (MSG_NOTE, vect_location,
14810 : "get vectype for scalar type: %T\n", scalar_type);
14811 : }
14812 4066208 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
14813 4066208 : if (!vectype)
14814 210979 : return opt_result::failure_at (stmt,
14815 : "not vectorized:"
14816 : " unsupported data-type %T\n",
14817 : scalar_type);
14818 :
14819 3855229 : if (dump_enabled_p ())
14820 62138 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14821 : }
14822 :
14823 4155618 : if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
14824 0 : return opt_result::failure_at (stmt,
14825 : "not vectorized: vector stmt in loop:%G",
14826 : stmt);
14827 :
14828 5683533 : *stmt_vectype_out = vectype;
14829 :
14830 : /* Don't try to compute scalar types if the stmt produces a boolean
14831 : vector; use the existing vector type instead. */
14832 5683533 : tree nunits_vectype = vectype;
14833 5683533 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14834 : {
14835 : /* The number of units is set according to the smallest scalar
14836 : type (or the largest vector size, but we only support one
14837 : vector size per vectorization). */
14838 5131442 : scalar_type = vect_get_smallest_scalar_type (stmt_info,
14839 5131442 : TREE_TYPE (vectype));
14840 5131442 : if (!types_compatible_p (scalar_type, TREE_TYPE (vectype)))
14841 : {
14842 1008799 : if (dump_enabled_p ())
14843 8327 : dump_printf_loc (MSG_NOTE, vect_location,
14844 : "get vectype for smallest scalar type: %T\n",
14845 : scalar_type);
14846 1008799 : nunits_vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
14847 : group_size);
14848 1008799 : if (!nunits_vectype)
14849 10 : return opt_result::failure_at
14850 10 : (stmt, "not vectorized: unsupported data-type %T\n",
14851 : scalar_type);
14852 1008789 : if (dump_enabled_p ())
14853 8327 : dump_printf_loc (MSG_NOTE, vect_location, "nunits vectype: %T\n",
14854 : nunits_vectype);
14855 : }
14856 : }
14857 :
14858 5683523 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (nunits_vectype),
14859 5683523 : TYPE_VECTOR_SUBPARTS (*stmt_vectype_out)))
14860 0 : return opt_result::failure_at (stmt,
14861 : "Not vectorized: Incompatible number "
14862 : "of vector subparts between %T and %T\n",
14863 : nunits_vectype, *stmt_vectype_out);
14864 :
14865 5683523 : if (dump_enabled_p ())
14866 : {
14867 146389 : dump_printf_loc (MSG_NOTE, vect_location, "nunits = ");
14868 146389 : dump_dec (MSG_NOTE, TYPE_VECTOR_SUBPARTS (nunits_vectype));
14869 146389 : dump_printf (MSG_NOTE, "\n");
14870 : }
14871 :
14872 5683523 : *nunits_vectype_out = nunits_vectype;
14873 5683523 : return opt_result::success ();
14874 : }
14875 :
14876 : /* Generate and return statement sequence that sets vector length LEN that is:
14877 :
14878 : min_of_start_and_end = min (START_INDEX, END_INDEX);
14879 : left_len = END_INDEX - min_of_start_and_end;
14880 : rhs = min (left_len, LEN_LIMIT);
14881 : LEN = rhs;
14882 :
14883 : Note: the cost of the code generated by this function is modeled
14884 : by vect_estimate_min_profitable_iters, so changes here may need
14885 : corresponding changes there. */
14886 :
14887 : gimple_seq
14888 0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
14889 : {
14890 0 : gimple_seq stmts = NULL;
14891 0 : tree len_type = TREE_TYPE (len);
14892 0 : gcc_assert (TREE_TYPE (start_index) == len_type);
14893 :
14894 0 : tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
14895 0 : tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
14896 0 : tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
14897 0 : gimple* stmt = gimple_build_assign (len, rhs);
14898 0 : gimple_seq_add_stmt (&stmts, stmt);
14899 :
14900 0 : return stmts;
14901 : }
14902 :
|
