Line data Source code
1 : /* Statement Analysis and Transformation for Vectorization
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "ssa.h"
31 : #include "optabs-tree.h"
32 : #include "insn-config.h"
33 : #include "recog.h" /* FIXME: for insn_data */
34 : #include "cgraph.h"
35 : #include "dumpfile.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "tree-eh.h"
40 : #include "gimplify.h"
41 : #include "gimple-iterator.h"
42 : #include "gimplify-me.h"
43 : #include "tree-cfg.h"
44 : #include "tree-ssa-loop-manip.h"
45 : #include "cfgloop.h"
46 : #include "explow.h"
47 : #include "tree-ssa-loop.h"
48 : #include "tree-scalar-evolution.h"
49 : #include "tree-vectorizer.h"
50 : #include "builtins.h"
51 : #include "internal-fn.h"
52 : #include "tree-vector-builder.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "tree-ssa-loop-niter.h"
56 : #include "gimple-fold.h"
57 : #include "regs.h"
58 : #include "attribs.h"
59 : #include "optabs-libfuncs.h"
60 : #include "tree-dfa.h"
61 :
62 : /* For lang_hooks.types.type_for_mode. */
63 : #include "langhooks.h"
64 :
65 : static tree vector_vector_composition_type (tree, poly_uint64, tree *,
66 : bool = false);
67 :
68 : /* Return TRUE iff the given statement is in an inner loop relative to
69 : the loop being vectorized. */
70 : bool
71 6121500 : stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
72 : {
73 6121500 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
74 6121500 : basic_block bb = gimple_bb (stmt);
75 6121500 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
76 2773145 : class loop* loop;
77 :
78 2773145 : if (!loop_vinfo)
79 : return false;
80 :
81 2773145 : loop = LOOP_VINFO_LOOP (loop_vinfo);
82 :
83 2773145 : return (bb->loop_father == loop->inner);
84 : }
85 :
86 : /* Record the cost of a statement, either by directly informing the
87 : target model or by saving it in a vector for later processing.
88 : Return a preliminary estimate of the statement's cost. */
89 :
90 : unsigned
91 9084528 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
92 : enum vect_cost_for_stmt kind,
93 : stmt_vec_info stmt_info, slp_tree node,
94 : tree vectype, int misalign,
95 : enum vect_cost_model_location where)
96 : {
97 9084528 : if ((kind == vector_load || kind == unaligned_load)
98 1655809 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
99 : kind = vector_gather_load;
100 9084528 : if ((kind == vector_store || kind == unaligned_store)
101 1044716 : && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
102 9084528 : kind = vector_scatter_store;
103 :
104 9084528 : stmt_info_for_cost si
105 9084528 : = { count, kind, where, stmt_info, node, vectype, misalign };
106 9084528 : body_cost_vec->safe_push (si);
107 :
108 9084528 : return (unsigned)
109 9084528 : (builtin_vectorization_cost (kind, vectype, misalign) * count);
110 : }
111 :
112 : unsigned
113 4235933 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
114 : enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
115 : tree vectype, int misalign,
116 : enum vect_cost_model_location where)
117 : {
118 4235933 : return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
119 4235933 : vectype, misalign, where);
120 : }
121 :
122 : unsigned
123 1709087 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
124 : enum vect_cost_for_stmt kind, slp_tree node,
125 : tree vectype, int misalign,
126 : enum vect_cost_model_location where)
127 : {
128 1709087 : return record_stmt_cost (body_cost_vec, count, kind,
129 : SLP_TREE_REPRESENTATIVE (node), node,
130 1709087 : vectype, misalign, where);
131 : }
132 :
133 : unsigned
134 0 : record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
135 : enum vect_cost_for_stmt kind,
136 : enum vect_cost_model_location where)
137 : {
138 0 : gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
139 : || kind == scalar_stmt);
140 0 : return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
141 0 : NULL_TREE, 0, where);
142 : }
143 :
144 : /* Return a variable of type ELEM_TYPE[NELEMS]. */
145 :
146 : static tree
147 0 : create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
148 : {
149 0 : return create_tmp_var (build_array_type_nelts (elem_type, nelems),
150 0 : "vect_array");
151 : }
152 :
153 : /* ARRAY is an array of vectors created by create_vector_array.
154 : Return an SSA_NAME for the vector in index N. The reference
155 : is part of the vectorization of STMT_INFO and the vector is associated
156 : with scalar destination SCALAR_DEST.
157 : If we need to ensure that inactive elements are set to zero,
158 : NEED_ZEROING is true, MASK contains the loop mask to be used. */
159 :
160 : static tree
161 0 : read_vector_array (vec_info *vinfo,
162 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
163 : tree scalar_dest, tree array, unsigned HOST_WIDE_INT n,
164 : bool need_zeroing, tree mask)
165 : {
166 0 : tree vect_type, vect, vect_name, tmp, tmp_name, array_ref;
167 0 : gimple *new_stmt;
168 :
169 0 : gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
170 0 : vect_type = TREE_TYPE (TREE_TYPE (array));
171 0 : tmp = vect_create_destination_var (scalar_dest, vect_type);
172 0 : vect = vect_create_destination_var (scalar_dest, vect_type);
173 0 : array_ref = build4 (ARRAY_REF, vect_type, array,
174 0 : build_int_cst (size_type_node, n),
175 : NULL_TREE, NULL_TREE);
176 :
177 0 : new_stmt = gimple_build_assign (tmp, array_ref);
178 0 : tmp_name = make_ssa_name (vect, new_stmt);
179 0 : gimple_assign_set_lhs (new_stmt, tmp_name);
180 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
181 :
182 0 : if (need_zeroing)
183 : {
184 0 : tree vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
185 : vect_type);
186 0 : vect_name = make_ssa_name (vect, new_stmt);
187 0 : new_stmt
188 0 : = gimple_build_assign (vect_name, VEC_COND_EXPR,
189 : mask, tmp_name, vec_els);
190 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
191 : }
192 : else
193 : vect_name = tmp_name;
194 :
195 0 : return vect_name;
196 : }
197 :
198 : /* ARRAY is an array of vectors created by create_vector_array.
199 : Emit code to store SSA_NAME VECT in index N of the array.
200 : The store is part of the vectorization of STMT_INFO. */
201 :
202 : static void
203 0 : write_vector_array (vec_info *vinfo,
204 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
205 : tree vect, tree array, unsigned HOST_WIDE_INT n)
206 : {
207 0 : tree array_ref;
208 0 : gimple *new_stmt;
209 :
210 0 : array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
211 0 : build_int_cst (size_type_node, n),
212 : NULL_TREE, NULL_TREE);
213 :
214 0 : new_stmt = gimple_build_assign (array_ref, vect);
215 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
216 0 : }
217 :
218 : /* PTR is a pointer to an array of type TYPE. Return a representation
219 : of *PTR. The memory reference replaces those in FIRST_DR
220 : (and its group). */
221 :
222 : static tree
223 0 : create_array_ref (tree type, tree ptr, tree alias_ptr_type)
224 : {
225 0 : tree mem_ref;
226 :
227 0 : mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
228 : /* Arrays have the same alignment as their type. */
229 0 : set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
230 0 : return mem_ref;
231 : }
232 :
233 : /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
234 : Emit the clobber before *GSI. */
235 :
236 : static void
237 15 : vect_clobber_variable (vec_info *vinfo, stmt_vec_info stmt_info,
238 : gimple_stmt_iterator *gsi, tree var)
239 : {
240 15 : tree clobber = build_clobber (TREE_TYPE (var));
241 15 : gimple *new_stmt = gimple_build_assign (var, clobber);
242 15 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
243 15 : }
244 :
245 : /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
246 :
247 : /* Function vect_mark_relevant.
248 :
249 : Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
250 :
251 : static void
252 3269448 : vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
253 : enum vect_relevant relevant, bool live_p)
254 : {
255 3269448 : enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
256 3269448 : bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
257 :
258 3269448 : if (dump_enabled_p ())
259 166153 : dump_printf_loc (MSG_NOTE, vect_location,
260 : "mark relevant %d, live %d: %G", relevant, live_p,
261 : stmt_info->stmt);
262 :
263 : /* If this stmt is an original stmt in a pattern, we might need to mark its
264 : related pattern stmt instead of the original stmt. However, such stmts
265 : may have their own uses that are not in any pattern, in such cases the
266 : stmt itself should be marked. */
267 3269448 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
268 : {
269 : /* This is the last stmt in a sequence that was detected as a
270 : pattern that can potentially be vectorized. Don't mark the stmt
271 : as relevant/live because it's not going to be vectorized.
272 : Instead mark the pattern-stmt that replaces it. */
273 :
274 252067 : if (dump_enabled_p ())
275 2866 : dump_printf_loc (MSG_NOTE, vect_location,
276 : "last stmt in pattern. don't mark"
277 : " relevant/live.\n");
278 :
279 252067 : stmt_vec_info old_stmt_info = stmt_info;
280 252067 : stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
281 252067 : gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
282 252067 : save_relevant = STMT_VINFO_RELEVANT (stmt_info);
283 252067 : save_live_p = STMT_VINFO_LIVE_P (stmt_info);
284 :
285 252067 : if (live_p && relevant == vect_unused_in_scope)
286 : {
287 112 : if (dump_enabled_p ())
288 10 : dump_printf_loc (MSG_NOTE, vect_location,
289 : "vec_stmt_relevant_p: forcing live pattern stmt "
290 : "relevant.\n");
291 : relevant = vect_used_only_live;
292 : }
293 :
294 252067 : if (dump_enabled_p ())
295 2866 : dump_printf_loc (MSG_NOTE, vect_location,
296 : "mark relevant %d, live %d: %G", relevant, live_p,
297 : stmt_info->stmt);
298 : }
299 :
300 3269448 : STMT_VINFO_LIVE_P (stmt_info) |= live_p;
301 3269448 : if (relevant > STMT_VINFO_RELEVANT (stmt_info))
302 2938767 : STMT_VINFO_RELEVANT (stmt_info) = relevant;
303 :
304 3269448 : if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
305 330681 : && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
306 : {
307 330022 : if (dump_enabled_p ())
308 20202 : dump_printf_loc (MSG_NOTE, vect_location,
309 : "already marked relevant/live.\n");
310 : return;
311 : }
312 :
313 2939426 : worklist->safe_push (stmt_info);
314 : }
315 :
316 :
317 : /* Function is_simple_and_all_uses_invariant
318 :
319 : Return true if STMT_INFO is simple and all uses of it are invariant. */
320 :
321 : bool
322 251198 : is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
323 : loop_vec_info loop_vinfo)
324 : {
325 251198 : tree op;
326 251198 : ssa_op_iter iter;
327 :
328 251198 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
329 195533 : if (!stmt)
330 : return false;
331 :
332 202826 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
333 : {
334 202055 : enum vect_def_type dt = vect_uninitialized_def;
335 :
336 202055 : if (!vect_is_simple_use (op, loop_vinfo, &dt))
337 : {
338 2175 : if (dump_enabled_p ())
339 22 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
340 : "use not simple.\n");
341 194762 : return false;
342 : }
343 :
344 199880 : if (dt != vect_external_def && dt != vect_constant_def)
345 : return false;
346 : }
347 : return true;
348 : }
349 :
350 : /* Function vect_stmt_relevant_p.
351 :
352 : Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
353 : is "relevant for vectorization".
354 :
355 : A stmt is considered "relevant for vectorization" if:
356 : - it has uses outside the loop.
357 : - it has vdefs (it alters memory).
358 : - control stmts in the loop (except for the exit condition).
359 :
360 : CHECKME: what other side effects would the vectorizer allow? */
361 :
362 : static bool
363 5291359 : vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
364 : enum vect_relevant *relevant, bool *live_p)
365 : {
366 5291359 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
367 5291359 : ssa_op_iter op_iter;
368 5291359 : imm_use_iterator imm_iter;
369 5291359 : use_operand_p use_p;
370 5291359 : def_operand_p def_p;
371 :
372 5291359 : *relevant = vect_unused_in_scope;
373 5291359 : *live_p = false;
374 :
375 : /* cond stmt other than loop exit cond. */
376 5291359 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
377 5291359 : if (is_ctrl_stmt (stmt)
378 626348 : && LOOP_VINFO_LOOP_IV_COND (loop_vinfo) != stmt
379 5529928 : && (!loop->inner || gimple_bb (stmt)->loop_father == loop))
380 236610 : *relevant = vect_used_in_scope;
381 :
382 : /* changing memory. */
383 5291359 : if (gimple_code (stmt_info->stmt) != GIMPLE_PHI)
384 4383163 : if (gimple_vdef (stmt_info->stmt)
385 3756815 : && !gimple_clobber_p (stmt_info->stmt))
386 : {
387 377104 : if (dump_enabled_p ())
388 28299 : dump_printf_loc (MSG_NOTE, vect_location,
389 : "vec_stmt_relevant_p: stmt has vdefs.\n");
390 377104 : *relevant = vect_used_in_scope;
391 377104 : if (! STMT_VINFO_DATA_REF (stmt_info)
392 377104 : && zero_ssa_operands (stmt_info->stmt, SSA_OP_DEF))
393 20 : LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo).safe_push (stmt_info);
394 : }
395 :
396 : /* uses outside the loop. */
397 14864039 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
398 : {
399 11482599 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
400 : {
401 7201278 : basic_block bb = gimple_bb (USE_STMT (use_p));
402 7201278 : if (!flow_bb_inside_loop_p (loop, bb))
403 : {
404 266413 : if (is_gimple_debug (USE_STMT (use_p)))
405 1107 : continue;
406 :
407 265306 : if (dump_enabled_p ())
408 6045 : dump_printf_loc (MSG_NOTE, vect_location,
409 : "vec_stmt_relevant_p: used out of loop.\n");
410 :
411 : /* We expect all such uses to be in the loop exit phis
412 : (because of loop closed form) */
413 265306 : gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
414 :
415 265306 : *live_p = true;
416 265306 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
417 : }
418 4281321 : }
419 : }
420 :
421 251200 : if (*live_p && *relevant == vect_unused_in_scope
422 5542557 : && !is_simple_and_all_uses_invariant (stmt_info, loop_vinfo))
423 : {
424 250427 : if (dump_enabled_p ())
425 5915 : dump_printf_loc (MSG_NOTE, vect_location,
426 : "vec_stmt_relevant_p: stmt live but not relevant.\n");
427 250427 : *relevant = vect_used_only_live;
428 : }
429 :
430 5291359 : return (*live_p || *relevant);
431 : }
432 :
433 :
434 : /* Function exist_non_indexing_operands_for_use_p
435 :
436 : USE is one of the uses attached to STMT_INFO. Check if USE is
437 : used in STMT_INFO for anything other than indexing an array. */
438 :
439 : static bool
440 4380817 : exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
441 : {
442 4380817 : tree operand;
443 :
444 : /* USE corresponds to some operand in STMT. If there is no data
445 : reference in STMT, then any operand that corresponds to USE
446 : is not indexing an array. */
447 4380817 : if (!STMT_VINFO_DATA_REF (stmt_info))
448 : return true;
449 :
450 : /* STMT has a data_ref. FORNOW this means that its of one of
451 : the following forms:
452 : -1- ARRAY_REF = var
453 : -2- var = ARRAY_REF
454 : (This should have been verified in analyze_data_refs).
455 :
456 : 'var' in the second case corresponds to a def, not a use,
457 : so USE cannot correspond to any operands that are not used
458 : for array indexing.
459 :
460 : Therefore, all we need to check is if STMT falls into the
461 : first case, and whether var corresponds to USE. */
462 :
463 1494664 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
464 1477918 : if (!assign || !gimple_assign_copy_p (assign))
465 : {
466 800834 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
467 16746 : if (call && gimple_call_internal_p (call))
468 : {
469 16746 : internal_fn ifn = gimple_call_internal_fn (call);
470 16746 : int mask_index = internal_fn_mask_index (ifn);
471 16746 : if (mask_index >= 0
472 16746 : && use == gimple_call_arg (call, mask_index))
473 : return true;
474 10930 : int els_index = internal_fn_else_index (ifn);
475 10930 : if (els_index >= 0
476 10930 : && use == gimple_call_arg (call, els_index))
477 : return true;
478 9546 : int stored_value_index = internal_fn_stored_value_index (ifn);
479 9546 : if (stored_value_index >= 0
480 9546 : && use == gimple_call_arg (call, stored_value_index))
481 : return true;
482 7396 : if (internal_gather_scatter_fn_p (ifn)
483 7396 : && use == gimple_call_arg (call, 1))
484 : return true;
485 : }
486 791484 : return false;
487 : }
488 :
489 693830 : if (TREE_CODE (gimple_assign_lhs (assign)) == SSA_NAME)
490 : return false;
491 693830 : operand = gimple_assign_rhs1 (assign);
492 693830 : if (TREE_CODE (operand) != SSA_NAME)
493 : return false;
494 :
495 599939 : if (operand == use)
496 283051 : return true;
497 :
498 : return false;
499 : }
500 :
501 :
502 : /*
503 : Function process_use.
504 :
505 : Inputs:
506 : - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
507 : - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
508 : that defined USE. This is done by calling mark_relevant and passing it
509 : the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
510 : - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
511 : be performed.
512 :
513 : Outputs:
514 : Generally, LIVE_P and RELEVANT are used to define the liveness and
515 : relevance info of the DEF_STMT of this USE:
516 : STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
517 : STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
518 : Exceptions:
519 : - case 1: If USE is used only for address computations (e.g. array indexing),
520 : which does not need to be directly vectorized, then the liveness/relevance
521 : of the respective DEF_STMT is left unchanged.
522 : - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
523 : we skip DEF_STMT cause it had already been processed.
524 : - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
525 : "relevant" will be modified accordingly.
526 :
527 : Return true if everything is as expected. Return false otherwise. */
528 :
529 : static opt_result
530 4439119 : process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
531 : enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
532 : bool force)
533 : {
534 4439119 : stmt_vec_info dstmt_vinfo;
535 4439119 : enum vect_def_type dt;
536 :
537 : /* case 1: we are only interested in uses that need to be vectorized. Uses
538 : that are used for address computation are not considered relevant. */
539 4439119 : if (!force && !exist_non_indexing_operands_for_use_p (use, stmt_vinfo))
540 1202263 : return opt_result::success ();
541 :
542 3236856 : if (!vect_is_simple_use (use, loop_vinfo, &dt, &dstmt_vinfo))
543 35567 : return opt_result::failure_at (stmt_vinfo->stmt,
544 : "not vectorized:"
545 : " unsupported use in stmt.\n");
546 :
547 3201289 : if (!dstmt_vinfo)
548 604623 : return opt_result::success ();
549 :
550 2596666 : basic_block def_bb = gimple_bb (dstmt_vinfo->stmt);
551 2596666 : basic_block bb = gimple_bb (stmt_vinfo->stmt);
552 :
553 : /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
554 : We have to force the stmt live since the epilogue loop needs it to
555 : continue computing the reduction. */
556 2596666 : if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
557 273785 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
558 85966 : && gimple_code (dstmt_vinfo->stmt) != GIMPLE_PHI
559 85966 : && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
560 2682632 : && bb->loop_father == def_bb->loop_father)
561 : {
562 85966 : if (dump_enabled_p ())
563 3986 : dump_printf_loc (MSG_NOTE, vect_location,
564 : "reduc-stmt defining reduc-phi in the same nest.\n");
565 85966 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, true);
566 85966 : return opt_result::success ();
567 : }
568 :
569 : /* case 3a: outer-loop stmt defining an inner-loop stmt:
570 : outer-loop-header-bb:
571 : d = dstmt_vinfo
572 : inner-loop:
573 : stmt # use (d)
574 : outer-loop-tail-bb:
575 : ... */
576 2510700 : if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
577 : {
578 2130 : if (dump_enabled_p ())
579 321 : dump_printf_loc (MSG_NOTE, vect_location,
580 : "outer-loop def-stmt defining inner-loop stmt.\n");
581 :
582 2130 : switch (relevant)
583 : {
584 0 : case vect_unused_in_scope:
585 0 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
586 : vect_used_in_scope : vect_unused_in_scope;
587 : break;
588 :
589 767 : case vect_used_in_outer_by_reduction:
590 767 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
591 : relevant = vect_used_by_reduction;
592 : break;
593 :
594 1083 : case vect_used_in_outer:
595 1083 : gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
596 : relevant = vect_used_in_scope;
597 : break;
598 :
599 : case vect_used_in_scope:
600 : break;
601 :
602 0 : default:
603 0 : gcc_unreachable ();
604 : }
605 : }
606 :
607 : /* case 3b: inner-loop stmt defining an outer-loop stmt:
608 : outer-loop-header-bb:
609 : ...
610 : inner-loop:
611 : d = dstmt_vinfo
612 : outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
613 : stmt # use (d) */
614 2508570 : else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
615 : {
616 2068 : if (dump_enabled_p ())
617 626 : dump_printf_loc (MSG_NOTE, vect_location,
618 : "inner-loop def-stmt defining outer-loop stmt.\n");
619 :
620 2068 : switch (relevant)
621 : {
622 0 : case vect_unused_in_scope:
623 2329051 : relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
624 0 : || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
625 : vect_used_in_outer_by_reduction : vect_unused_in_scope;
626 : break;
627 :
628 : case vect_used_by_reduction:
629 : case vect_used_only_live:
630 : relevant = vect_used_in_outer_by_reduction;
631 : break;
632 :
633 : case vect_used_in_scope:
634 2329051 : relevant = vect_used_in_outer;
635 : break;
636 :
637 0 : default:
638 0 : gcc_unreachable ();
639 : }
640 : }
641 : /* We are also not interested in uses on loop PHI backedges that are
642 : inductions. Otherwise we'll needlessly vectorize the IV increment
643 : and cause hybrid SLP for SLP inductions. */
644 2506502 : else if (gimple_code (stmt_vinfo->stmt) == GIMPLE_PHI
645 184615 : && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
646 2688151 : && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo->stmt,
647 : loop_latch_edge (bb->loop_father))
648 : == use))
649 : {
650 181649 : if (dump_enabled_p ())
651 4927 : dump_printf_loc (MSG_NOTE, vect_location,
652 : "induction value on backedge.\n");
653 181649 : return opt_result::success ();
654 : }
655 :
656 2329051 : vect_mark_relevant (worklist, dstmt_vinfo, relevant, false);
657 2329051 : return opt_result::success ();
658 : }
659 :
660 :
661 : /* Function vect_mark_stmts_to_be_vectorized.
662 :
663 : Not all stmts in the loop need to be vectorized. For example:
664 :
665 : for i...
666 : for j...
667 : 1. T0 = i + j
668 : 2. T1 = a[T0]
669 :
670 : 3. j = j + 1
671 :
672 : Stmt 1 and 3 do not need to be vectorized, because loop control and
673 : addressing of vectorized data-refs are handled differently.
674 :
675 : This pass detects such stmts. */
676 :
677 : opt_result
678 445418 : vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
679 : {
680 445418 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
681 445418 : basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
682 445418 : unsigned int nbbs = loop->num_nodes;
683 445418 : gimple_stmt_iterator si;
684 445418 : unsigned int i;
685 445418 : basic_block bb;
686 445418 : bool live_p;
687 445418 : enum vect_relevant relevant;
688 :
689 445418 : DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
690 :
691 445418 : auto_vec<stmt_vec_info, 64> worklist;
692 :
693 : /* 1. Init worklist. */
694 1510488 : for (i = 0; i < nbbs; i++)
695 : {
696 1075696 : bb = bbs[i];
697 2207969 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
698 : {
699 2285508 : if (virtual_operand_p (gimple_phi_result (gsi_stmt (si))))
700 234558 : continue;
701 908196 : stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
702 908196 : if (dump_enabled_p ())
703 42085 : dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? %G",
704 : phi_info->stmt);
705 :
706 908196 : if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
707 : {
708 46793 : if (STMT_VINFO_DEF_TYPE (phi_info) == vect_unknown_def_type)
709 10481 : return opt_result::failure_at
710 10481 : (*si, "not vectorized: unhandled relevant PHI: %G", *si);
711 36312 : vect_mark_relevant (&worklist, phi_info, relevant, live_p);
712 : }
713 : }
714 8593618 : for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si))
715 : {
716 7528548 : gimple *stmt = gsi_stmt (si);
717 7528548 : if (is_gimple_debug (stmt))
718 3145240 : continue;
719 4383308 : stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
720 4383308 : if (dump_enabled_p ())
721 226505 : dump_printf_loc (MSG_NOTE, vect_location,
722 : "init: stmt relevant? %G", stmt);
723 :
724 4383308 : if (gimple_get_lhs (stmt) == NULL_TREE
725 632576 : && !is_a <gcond *> (stmt)
726 4389536 : && !is_a <gcall *> (stmt))
727 145 : return opt_result::failure_at
728 145 : (stmt, "not vectorized: irregular stmt: %G", stmt);
729 :
730 4383163 : if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
731 818119 : vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
732 : }
733 : }
734 :
735 : /* 2. Process_worklist */
736 3243080 : while (worklist.length () > 0)
737 : {
738 2843857 : use_operand_p use_p;
739 2843857 : ssa_op_iter iter;
740 :
741 2843857 : stmt_vec_info stmt_vinfo = worklist.pop ();
742 2843857 : if (dump_enabled_p ())
743 145290 : dump_printf_loc (MSG_NOTE, vect_location,
744 : "worklist: examine stmt: %G", stmt_vinfo->stmt);
745 :
746 : /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
747 : (DEF_STMT) as relevant/irrelevant according to the relevance property
748 : of STMT. */
749 2843857 : relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
750 :
751 : /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
752 : propagated as is to the DEF_STMTs of its USEs.
753 :
754 : One exception is when STMT has been identified as defining a reduction
755 : variable; in this case we set the relevance to vect_used_by_reduction.
756 : This is because we distinguish between two kinds of relevant stmts -
757 : those that are used by a reduction computation, and those that are
758 : (also) used by a regular computation. This allows us later on to
759 : identify stmts that are used solely by a reduction, and therefore the
760 : order of the results that they produce does not have to be kept. */
761 :
762 2843857 : switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
763 : {
764 174897 : case vect_reduction_def:
765 174897 : gcc_assert (relevant != vect_unused_in_scope);
766 174897 : if (relevant != vect_unused_in_scope
767 174897 : && relevant != vect_used_in_scope
768 174897 : && relevant != vect_used_by_reduction
769 174897 : && relevant != vect_used_only_live)
770 0 : return opt_result::failure_at
771 0 : (stmt_vinfo->stmt, "unsupported use of reduction.\n");
772 : break;
773 :
774 2089 : case vect_nested_cycle:
775 2089 : if (relevant != vect_unused_in_scope
776 2089 : && relevant != vect_used_in_outer_by_reduction
777 1494 : && relevant != vect_used_in_outer)
778 2 : return opt_result::failure_at
779 2 : (stmt_vinfo->stmt, "unsupported use of nested cycle.\n");
780 : break;
781 :
782 1209 : case vect_double_reduction_def:
783 1209 : if (relevant != vect_unused_in_scope
784 1209 : && relevant != vect_used_by_reduction
785 409 : && relevant != vect_used_only_live)
786 0 : return opt_result::failure_at
787 0 : (stmt_vinfo->stmt, "unsupported use of double reduction.\n");
788 : break;
789 :
790 : default:
791 : break;
792 : }
793 :
794 2843855 : if (is_pattern_stmt_p (stmt_vinfo))
795 : {
796 : /* Pattern statements are not inserted into the code, so
797 : FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
798 : have to scan the RHS or function arguments instead. */
799 643325 : if (gassign *assign = dyn_cast <gassign *> (stmt_vinfo->stmt))
800 : {
801 416934 : enum tree_code rhs_code = gimple_assign_rhs_code (assign);
802 416934 : tree op = gimple_assign_rhs1 (assign);
803 :
804 416934 : i = 1;
805 416934 : if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
806 : {
807 0 : opt_result res
808 0 : = process_use (stmt_vinfo, TREE_OPERAND (op, 0),
809 : loop_vinfo, relevant, &worklist, false);
810 0 : if (!res)
811 0 : return res;
812 0 : res = process_use (stmt_vinfo, TREE_OPERAND (op, 1),
813 : loop_vinfo, relevant, &worklist, false);
814 0 : if (!res)
815 0 : return res;
816 416934 : i = 2;
817 : }
818 1201907 : for (; i < gimple_num_ops (assign); i++)
819 : {
820 788848 : op = gimple_op (assign, i);
821 788848 : if (TREE_CODE (op) == SSA_NAME)
822 : {
823 597506 : opt_result res
824 597506 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
825 : &worklist, false);
826 597506 : if (!res)
827 3875 : return res;
828 : }
829 : }
830 : }
831 226391 : else if (gcond *cond = dyn_cast <gcond *> (stmt_vinfo->stmt))
832 : {
833 219569 : tree_code rhs_code = gimple_cond_code (cond);
834 219569 : gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
835 219569 : opt_result res
836 219569 : = process_use (stmt_vinfo, gimple_cond_lhs (cond),
837 : loop_vinfo, relevant, &worklist, false);
838 219569 : if (!res)
839 35569 : return res;
840 219569 : res = process_use (stmt_vinfo, gimple_cond_rhs (cond),
841 : loop_vinfo, relevant, &worklist, false);
842 219569 : if (!res)
843 0 : return res;
844 : }
845 6822 : else if (gcall *call = dyn_cast <gcall *> (stmt_vinfo->stmt))
846 : {
847 32014 : for (i = 0; i < gimple_call_num_args (call); i++)
848 : {
849 25192 : tree arg = gimple_call_arg (call, i);
850 25192 : opt_result res
851 25192 : = process_use (stmt_vinfo, arg, loop_vinfo, relevant,
852 : &worklist, false);
853 25192 : if (!res)
854 0 : return res;
855 : }
856 : }
857 : else
858 0 : gcc_unreachable ();
859 : }
860 : else
861 7702172 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt_vinfo->stmt, iter, SSA_OP_USE)
862 : {
863 3318981 : tree op = USE_FROM_PTR (use_p);
864 3318981 : opt_result res
865 3318981 : = process_use (stmt_vinfo, op, loop_vinfo, relevant,
866 : &worklist, false);
867 3318981 : if (!res)
868 17869 : return res;
869 : }
870 :
871 2822111 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
872 : {
873 58302 : gather_scatter_info gs_info;
874 58302 : if (!vect_check_gather_scatter (stmt_vinfo,
875 : STMT_VINFO_VECTYPE (stmt_vinfo),
876 : loop_vinfo, &gs_info))
877 0 : gcc_unreachable ();
878 58302 : opt_result res
879 58302 : = process_use (stmt_vinfo, gs_info.offset, loop_vinfo, relevant,
880 : &worklist, true);
881 58302 : if (!res)
882 : {
883 13823 : if (fatal)
884 13823 : *fatal = false;
885 13823 : return res;
886 : }
887 : }
888 : } /* while worklist */
889 :
890 399223 : return opt_result::success ();
891 445418 : }
892 :
893 : /* Function vect_model_simple_cost.
894 :
895 : Models cost for simple operations, i.e. those that only emit N operations
896 : of the same KIND. */
897 :
898 : static void
899 669271 : vect_model_simple_cost (vec_info *vinfo, int n, slp_tree node,
900 : stmt_vector_for_cost *cost_vec,
901 : vect_cost_for_stmt kind = vector_stmt)
902 : {
903 669271 : int inside_cost = 0, prologue_cost = 0;
904 :
905 669271 : gcc_assert (cost_vec != NULL);
906 :
907 669271 : n *= vect_get_num_copies (vinfo, node);
908 :
909 : /* Pass the inside-of-loop statements to the target-specific cost model. */
910 669271 : inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
911 :
912 669271 : if (dump_enabled_p ())
913 34307 : dump_printf_loc (MSG_NOTE, vect_location,
914 : "vect_model_simple_cost: inside_cost = %d, "
915 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
916 669271 : }
917 :
918 :
919 : /* Model cost for type demotion and promotion operations. PWR is
920 : normally zero for single-step promotions and demotions. It will be
921 : one if two-step promotion/demotion is required, and so on. NCOPIES
922 : is the number of vector results (and thus number of instructions)
923 : for the narrowest end of the operation chain. Each additional
924 : step doubles the number of instructions required. If WIDEN_ARITH
925 : is true the stmt is doing widening arithmetic. */
926 :
927 : static void
928 68683 : vect_model_promotion_demotion_cost (slp_tree slp_node,
929 : unsigned int ncopies, int pwr,
930 : stmt_vector_for_cost *cost_vec,
931 : bool widen_arith)
932 : {
933 68683 : int i;
934 68683 : int inside_cost = 0, prologue_cost = 0;
935 :
936 160342 : for (i = 0; i < pwr + 1; i++)
937 : {
938 181427 : inside_cost += record_stmt_cost (cost_vec, ncopies,
939 : widen_arith
940 : ? vector_stmt : vec_promote_demote,
941 : slp_node, 0, vect_body);
942 91659 : ncopies *= 2;
943 : }
944 :
945 68683 : if (dump_enabled_p ())
946 6220 : dump_printf_loc (MSG_NOTE, vect_location,
947 : "vect_model_promotion_demotion_cost: inside_cost = %d, "
948 : "prologue_cost = %d .\n", inside_cost, prologue_cost);
949 68683 : }
950 :
951 : /* Returns true if the current function returns DECL. */
952 :
953 : static bool
954 569267 : cfun_returns (tree decl)
955 : {
956 569267 : edge_iterator ei;
957 569267 : edge e;
958 1121018 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
959 : {
960 1127174 : greturn *ret = safe_dyn_cast <greturn *> (*gsi_last_bb (e->src));
961 563587 : if (!ret)
962 0 : continue;
963 563587 : if (gimple_return_retval (ret) == decl)
964 : return true;
965 : /* We often end up with an aggregate copy to the result decl,
966 : handle that case as well. First skip intermediate clobbers
967 : though. */
968 : gimple *def = ret;
969 1705282 : do
970 : {
971 3410564 : def = SSA_NAME_DEF_STMT (gimple_vuse (def));
972 : }
973 1705282 : while (gimple_clobber_p (def));
974 552548 : if (is_a <gassign *> (def)
975 63267 : && gimple_assign_lhs (def) == gimple_return_retval (ret)
976 559629 : && gimple_assign_rhs1 (def) == decl)
977 : return true;
978 : }
979 : return false;
980 : }
981 :
982 : /* Calculate cost of DR's memory access. */
983 : void
984 1034644 : vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
985 : int ncopies, dr_alignment_support alignment_support_scheme,
986 : int misalignment,
987 : unsigned int *inside_cost,
988 : stmt_vector_for_cost *body_cost_vec)
989 : {
990 1034644 : tree vectype
991 1034644 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
992 1034644 : switch (alignment_support_scheme)
993 : {
994 568235 : case dr_aligned:
995 568235 : {
996 568235 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
997 : vector_store, stmt_info, slp_node,
998 : vectype, 0, vect_body);
999 :
1000 568235 : if (dump_enabled_p ())
1001 14727 : dump_printf_loc (MSG_NOTE, vect_location,
1002 : "vect_model_store_cost: aligned.\n");
1003 : break;
1004 : }
1005 :
1006 466409 : case dr_unaligned_supported:
1007 466409 : {
1008 : /* Here, we assign an additional cost for the unaligned store. */
1009 466409 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1010 : unaligned_store, stmt_info, slp_node,
1011 : vectype, misalignment, vect_body);
1012 466409 : if (dump_enabled_p ())
1013 13294 : dump_printf_loc (MSG_NOTE, vect_location,
1014 : "vect_model_store_cost: unaligned supported by "
1015 : "hardware.\n");
1016 : break;
1017 : }
1018 :
1019 0 : case dr_unaligned_unsupported:
1020 0 : {
1021 0 : *inside_cost = VECT_MAX_COST;
1022 :
1023 0 : if (dump_enabled_p ())
1024 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1025 : "vect_model_store_cost: unsupported access.\n");
1026 : break;
1027 : }
1028 :
1029 0 : default:
1030 0 : gcc_unreachable ();
1031 : }
1032 1034644 : }
1033 :
1034 : /* Calculate cost of DR's memory access. */
1035 : void
1036 980733 : vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, slp_tree slp_node,
1037 : int ncopies, dr_alignment_support alignment_support_scheme,
1038 : int misalignment,
1039 : bool add_realign_cost, unsigned int *inside_cost,
1040 : unsigned int *prologue_cost,
1041 : stmt_vector_for_cost *prologue_cost_vec,
1042 : stmt_vector_for_cost *body_cost_vec,
1043 : bool record_prologue_costs)
1044 : {
1045 980733 : tree vectype
1046 980733 : = slp_node ? SLP_TREE_VECTYPE (slp_node) : STMT_VINFO_VECTYPE (stmt_info);
1047 980733 : switch (alignment_support_scheme)
1048 : {
1049 564042 : case dr_aligned:
1050 564042 : {
1051 564042 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1052 : stmt_info, slp_node, vectype,
1053 : 0, vect_body);
1054 :
1055 564042 : if (dump_enabled_p ())
1056 19268 : dump_printf_loc (MSG_NOTE, vect_location,
1057 : "vect_model_load_cost: aligned.\n");
1058 :
1059 : break;
1060 : }
1061 356387 : case dr_unaligned_supported:
1062 356387 : {
1063 : /* Here, we assign an additional cost for the unaligned load. */
1064 356387 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1065 : unaligned_load, stmt_info, slp_node,
1066 : vectype, misalignment, vect_body);
1067 :
1068 356387 : if (dump_enabled_p ())
1069 23114 : dump_printf_loc (MSG_NOTE, vect_location,
1070 : "vect_model_load_cost: unaligned supported by "
1071 : "hardware.\n");
1072 :
1073 : break;
1074 : }
1075 0 : case dr_explicit_realign:
1076 0 : {
1077 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1078 : vector_load, stmt_info, slp_node,
1079 : vectype, 0, vect_body);
1080 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1081 : vec_perm, stmt_info, slp_node,
1082 : vectype, 0, vect_body);
1083 :
1084 : /* FIXME: If the misalignment remains fixed across the iterations of
1085 : the containing loop, the following cost should be added to the
1086 : prologue costs. */
1087 0 : if (targetm.vectorize.builtin_mask_for_load)
1088 0 : *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1089 : stmt_info, slp_node, vectype,
1090 : 0, vect_body);
1091 :
1092 0 : if (dump_enabled_p ())
1093 0 : dump_printf_loc (MSG_NOTE, vect_location,
1094 : "vect_model_load_cost: explicit realign\n");
1095 :
1096 : break;
1097 : }
1098 0 : case dr_explicit_realign_optimized:
1099 0 : {
1100 0 : if (dump_enabled_p ())
1101 0 : dump_printf_loc (MSG_NOTE, vect_location,
1102 : "vect_model_load_cost: unaligned software "
1103 : "pipelined.\n");
1104 :
1105 : /* Unaligned software pipeline has a load of an address, an initial
1106 : load, and possibly a mask operation to "prime" the loop. However,
1107 : if this is an access in a group of loads, which provide grouped
1108 : access, then the above cost should only be considered for one
1109 : access in the group. Inside the loop, there is a load op
1110 : and a realignment op. */
1111 :
1112 0 : if (add_realign_cost && record_prologue_costs)
1113 : {
1114 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1115 : vector_stmt, stmt_info,
1116 : slp_node, vectype,
1117 : 0, vect_prologue);
1118 0 : if (targetm.vectorize.builtin_mask_for_load)
1119 0 : *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1120 : vector_stmt, stmt_info,
1121 : slp_node, vectype,
1122 : 0, vect_prologue);
1123 : }
1124 :
1125 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1126 : stmt_info, slp_node, vectype,
1127 : 0, vect_body);
1128 0 : *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1129 : stmt_info, slp_node, vectype,
1130 : 0, vect_body);
1131 :
1132 0 : if (dump_enabled_p ())
1133 0 : dump_printf_loc (MSG_NOTE, vect_location,
1134 : "vect_model_load_cost: explicit realign optimized"
1135 : "\n");
1136 :
1137 : break;
1138 : }
1139 :
1140 60304 : case dr_unaligned_unsupported:
1141 60304 : {
1142 60304 : *inside_cost = VECT_MAX_COST;
1143 :
1144 60304 : if (dump_enabled_p ())
1145 116 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1146 : "vect_model_load_cost: unsupported access.\n");
1147 : break;
1148 : }
1149 :
1150 0 : default:
1151 0 : gcc_unreachable ();
1152 : }
1153 980733 : }
1154 :
1155 : /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1156 : the loop preheader for the vectorized stmt STMT_VINFO. */
1157 :
1158 : static void
1159 6682 : vect_init_vector_1 (vec_info *vinfo, stmt_vec_info stmt_vinfo, gimple *new_stmt,
1160 : gimple_stmt_iterator *gsi)
1161 : {
1162 6682 : if (gsi)
1163 3403 : vect_finish_stmt_generation (vinfo, stmt_vinfo, new_stmt, gsi);
1164 : else
1165 3279 : vinfo->insert_on_entry (stmt_vinfo, new_stmt);
1166 :
1167 6682 : if (dump_enabled_p ())
1168 1854 : dump_printf_loc (MSG_NOTE, vect_location,
1169 : "created new init_stmt: %G", new_stmt);
1170 6682 : }
1171 :
1172 : /* Function vect_init_vector.
1173 :
1174 : Insert a new stmt (INIT_STMT) that initializes a new variable of type
1175 : TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1176 : vector type a vector with all elements equal to VAL is created first.
1177 : Place the initialization at GSI if it is not NULL. Otherwise, place the
1178 : initialization at the loop preheader.
1179 : Return the DEF of INIT_STMT.
1180 : It will be used in the vectorization of STMT_INFO. */
1181 :
1182 : tree
1183 4965 : vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
1184 : gimple_stmt_iterator *gsi)
1185 : {
1186 4965 : gimple *init_stmt;
1187 4965 : tree new_temp;
1188 :
1189 : /* We abuse this function to push sth to a SSA name with initial 'val'. */
1190 4965 : if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1191 : {
1192 1314 : gcc_assert (VECTOR_TYPE_P (type));
1193 1314 : if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1194 : {
1195 : /* Scalar boolean value should be transformed into
1196 : all zeros or all ones value before building a vector. */
1197 11 : if (VECTOR_BOOLEAN_TYPE_P (type))
1198 : {
1199 3 : tree true_val = build_all_ones_cst (TREE_TYPE (type));
1200 3 : tree false_val = build_zero_cst (TREE_TYPE (type));
1201 :
1202 3 : if (CONSTANT_CLASS_P (val))
1203 0 : val = integer_zerop (val) ? false_val : true_val;
1204 : else
1205 : {
1206 3 : new_temp = make_ssa_name (TREE_TYPE (type));
1207 3 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1208 : val, true_val, false_val);
1209 3 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1210 3 : val = new_temp;
1211 : }
1212 : }
1213 : else
1214 : {
1215 8 : gimple_seq stmts = NULL;
1216 8 : if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1217 8 : val = gimple_build (&stmts, VIEW_CONVERT_EXPR,
1218 8 : TREE_TYPE (type), val);
1219 : else
1220 : /* ??? Condition vectorization expects us to do
1221 : promotion of invariant/external defs. */
1222 0 : val = gimple_convert (&stmts, TREE_TYPE (type), val);
1223 16 : for (gimple_stmt_iterator gsi2 = gsi_start (stmts);
1224 16 : !gsi_end_p (gsi2); )
1225 : {
1226 8 : init_stmt = gsi_stmt (gsi2);
1227 8 : gsi_remove (&gsi2, false);
1228 8 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1229 : }
1230 : }
1231 : }
1232 1314 : val = build_vector_from_val (type, val);
1233 : }
1234 :
1235 4965 : new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1236 4965 : init_stmt = gimple_build_assign (new_temp, val);
1237 4965 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, gsi);
1238 4965 : return new_temp;
1239 : }
1240 :
1241 :
1242 : /* Get vectorized definitions for OP0 and OP1. */
1243 :
1244 : void
1245 193428 : vect_get_vec_defs (vec_info *, slp_tree slp_node,
1246 : bool op0, vec<tree> *vec_oprnds0,
1247 : bool op1, vec<tree> *vec_oprnds1,
1248 : bool op2, vec<tree> *vec_oprnds2,
1249 : bool op3, vec<tree> *vec_oprnds3)
1250 : {
1251 193428 : if (op0)
1252 191763 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
1253 193428 : if (op1)
1254 141332 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
1255 193428 : if (op2)
1256 9408 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
1257 193428 : if (op3)
1258 0 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
1259 193428 : }
1260 :
1261 : /* Helper function called by vect_finish_replace_stmt and
1262 : vect_finish_stmt_generation. Set the location of the new
1263 : statement and create and return a stmt_vec_info for it. */
1264 :
1265 : static void
1266 1467712 : vect_finish_stmt_generation_1 (vec_info *,
1267 : stmt_vec_info stmt_info, gimple *vec_stmt)
1268 : {
1269 1467712 : if (dump_enabled_p ())
1270 148473 : dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt);
1271 :
1272 1467712 : if (stmt_info)
1273 : {
1274 1438857 : gimple_set_location (vec_stmt, gimple_location (stmt_info->stmt));
1275 :
1276 : /* While EH edges will generally prevent vectorization, stmt might
1277 : e.g. be in a must-not-throw region. Ensure newly created stmts
1278 : that could throw are part of the same region. */
1279 1438857 : int lp_nr = lookup_stmt_eh_lp (stmt_info->stmt);
1280 1438857 : if (lp_nr != 0 && stmt_could_throw_p (cfun, vec_stmt))
1281 48 : add_stmt_to_eh_lp (vec_stmt, lp_nr);
1282 : }
1283 : else
1284 28855 : gcc_assert (!stmt_could_throw_p (cfun, vec_stmt));
1285 1467712 : }
1286 :
1287 : /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1288 : which sets the same scalar result as STMT_INFO did. Create and return a
1289 : stmt_vec_info for VEC_STMT. */
1290 :
1291 : void
1292 927 : vect_finish_replace_stmt (vec_info *vinfo,
1293 : stmt_vec_info stmt_info, gimple *vec_stmt)
1294 : {
1295 927 : gimple *scalar_stmt = vect_orig_stmt (stmt_info)->stmt;
1296 927 : gcc_assert (gimple_get_lhs (scalar_stmt) == gimple_get_lhs (vec_stmt));
1297 :
1298 927 : gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt);
1299 927 : gsi_replace (&gsi, vec_stmt, true);
1300 :
1301 927 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1302 927 : }
1303 :
1304 : /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1305 : before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1306 :
1307 : void
1308 1466785 : vect_finish_stmt_generation (vec_info *vinfo,
1309 : stmt_vec_info stmt_info, gimple *vec_stmt,
1310 : gimple_stmt_iterator *gsi)
1311 : {
1312 1466785 : gcc_assert (!stmt_info || gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
1313 :
1314 1466785 : if (!gsi_end_p (*gsi)
1315 2932613 : && gimple_has_mem_ops (vec_stmt))
1316 : {
1317 1465828 : gimple *at_stmt = gsi_stmt (*gsi);
1318 1465828 : tree vuse = gimple_vuse (at_stmt);
1319 1459723 : if (vuse && TREE_CODE (vuse) == SSA_NAME)
1320 : {
1321 1313233 : tree vdef = gimple_vdef (at_stmt);
1322 1313233 : gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1323 1313233 : gimple_set_modified (vec_stmt, true);
1324 : /* If we have an SSA vuse and insert a store, update virtual
1325 : SSA form to avoid triggering the renamer. Do so only
1326 : if we can easily see all uses - which is what almost always
1327 : happens with the way vectorized stmts are inserted. */
1328 766996 : if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1329 2080193 : && ((is_gimple_assign (vec_stmt)
1330 766123 : && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1331 64300 : || (is_gimple_call (vec_stmt)
1332 837 : && (!(gimple_call_flags (vec_stmt)
1333 837 : & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
1334 3 : || (gimple_call_lhs (vec_stmt)
1335 3 : && !is_gimple_reg (gimple_call_lhs (vec_stmt)))))))
1336 : {
1337 703494 : tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1338 703494 : gimple_set_vdef (vec_stmt, new_vdef);
1339 703494 : SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1340 : }
1341 : }
1342 : }
1343 1466785 : gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1344 1466785 : vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt);
1345 1466785 : }
1346 :
1347 : /* We want to vectorize a call to combined function CFN with function
1348 : decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1349 : as the types of all inputs. Check whether this is possible using
1350 : an internal function, returning its code if so or IFN_LAST if not. */
1351 :
1352 : static internal_fn
1353 16760 : vectorizable_internal_function (combined_fn cfn, tree fndecl,
1354 : tree vectype_out, tree vectype_in)
1355 : {
1356 16760 : internal_fn ifn;
1357 16760 : if (internal_fn_p (cfn))
1358 14155 : ifn = as_internal_fn (cfn);
1359 : else
1360 2605 : ifn = associated_internal_fn (fndecl);
1361 16760 : if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1362 : {
1363 13346 : const direct_internal_fn_info &info = direct_internal_fn (ifn);
1364 13346 : if (info.vectorizable)
1365 : {
1366 13346 : bool same_size_p = TYPE_SIZE (vectype_in) == TYPE_SIZE (vectype_out);
1367 13346 : tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1368 13346 : tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1369 :
1370 : /* The type size of both the vectype_in and vectype_out should be
1371 : exactly the same when vectype_out isn't participating the optab.
1372 : While there is no restriction for type size when vectype_out
1373 : is part of the optab query. */
1374 13346 : if (type0 != vectype_out && type1 != vectype_out && !same_size_p)
1375 16760 : return IFN_LAST;
1376 :
1377 13309 : if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1378 : OPTIMIZE_FOR_SPEED))
1379 7835 : return ifn;
1380 : }
1381 : }
1382 : return IFN_LAST;
1383 : }
1384 :
1385 :
1386 : static tree permute_vec_elements (vec_info *, tree, tree, tree, stmt_vec_info,
1387 : gimple_stmt_iterator *);
1388 :
1389 : /* Check whether a load or store statement in the loop described by
1390 : LOOP_VINFO is possible in a loop using partial vectors. This is
1391 : testing whether the vectorizer pass has the appropriate support,
1392 : as well as whether the target does.
1393 :
1394 : VLS_TYPE says whether the statement is a load or store and VECTYPE
1395 : is the type of the vector being loaded or stored. SLP_NODE is the SLP
1396 : node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1397 : says how the load or store is going to be implemented and GROUP_SIZE
1398 : is the number of load or store statements in the containing group.
1399 : If the access is a gather load or scatter store, GS_INFO describes
1400 : its arguments. If the load or store is conditional, SCALAR_MASK is the
1401 : condition under which it occurs.
1402 :
1403 : Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1404 : vectors is not supported, otherwise record the required rgroup control
1405 : types.
1406 :
1407 : If partial vectors can be used and ELSVALS is nonzero the supported
1408 : else values will be added to the vector ELSVALS points to. */
1409 :
1410 : static void
1411 297457 : check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
1412 : slp_tree slp_node,
1413 : vec_load_store_type vls_type,
1414 : int group_size,
1415 : vect_load_store_data *ls,
1416 : slp_tree mask_node,
1417 : vec<int> *elsvals = nullptr)
1418 : {
1419 297457 : vect_memory_access_type memory_access_type = ls->memory_access_type;
1420 :
1421 : /* Invariant loads need no special support. */
1422 297457 : if (memory_access_type == VMAT_INVARIANT)
1423 29483 : return;
1424 :
1425 : /* Figure whether the mask is uniform. scalar_mask is used to
1426 : populate the scalar_cond_masked_set. */
1427 296265 : tree scalar_mask = NULL_TREE;
1428 296265 : if (mask_node)
1429 4906 : for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
1430 : {
1431 2484 : tree def = vect_get_slp_scalar_def (mask_node, i);
1432 2484 : if (!def
1433 2484 : || (scalar_mask && def != scalar_mask))
1434 : {
1435 : scalar_mask = NULL;
1436 : break;
1437 : }
1438 : else
1439 2463 : scalar_mask = def;
1440 : }
1441 :
1442 296265 : unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node);
1443 296265 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1444 296265 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
1445 296265 : machine_mode vecmode = TYPE_MODE (vectype);
1446 296265 : bool is_load = (vls_type == VLS_LOAD);
1447 296265 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
1448 : {
1449 0 : nvectors /= group_size;
1450 0 : internal_fn ifn
1451 0 : = (is_load ? vect_load_lanes_supported (vectype, group_size, true,
1452 : elsvals)
1453 0 : : vect_store_lanes_supported (vectype, group_size, true));
1454 0 : if (ifn == IFN_MASK_LEN_LOAD_LANES || ifn == IFN_MASK_LEN_STORE_LANES)
1455 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1456 0 : else if (ifn == IFN_MASK_LOAD_LANES || ifn == IFN_MASK_STORE_LANES)
1457 0 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1458 : scalar_mask);
1459 : else
1460 : {
1461 0 : if (dump_enabled_p ())
1462 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1463 : "can't operate on partial vectors because"
1464 : " the target doesn't have an appropriate"
1465 : " load/store-lanes instruction.\n");
1466 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1467 : }
1468 : return;
1469 : }
1470 :
1471 296265 : if (mat_gather_scatter_p (memory_access_type))
1472 : {
1473 1764 : internal_fn ifn = (is_load
1474 1764 : ? IFN_MASK_GATHER_LOAD
1475 : : IFN_MASK_SCATTER_STORE);
1476 418 : internal_fn len_ifn = (is_load
1477 : ? IFN_MASK_LEN_GATHER_LOAD
1478 : : IFN_MASK_LEN_SCATTER_STORE);
1479 1764 : stmt_vec_info repr = SLP_TREE_REPRESENTATIVE (slp_node);
1480 1764 : tree off_vectype = (STMT_VINFO_GATHER_SCATTER_P (repr)
1481 1764 : ? SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0])
1482 1764 : : ls->strided_offset_vectype);
1483 1764 : tree memory_type = TREE_TYPE (DR_REF (STMT_VINFO_DR_INFO (repr)->dr));
1484 1764 : int scale = SLP_TREE_GS_SCALE (slp_node);
1485 :
1486 : /* The following "supported" checks just verify what we established in
1487 : get_load_store_type and don't try different offset types.
1488 : Therefore, off_vectype must be a supported offset type. In case
1489 : we chose a different one use this instead. */
1490 1764 : if (ls->supported_offset_vectype)
1491 0 : off_vectype = ls->supported_offset_vectype;
1492 : /* Same for scale. */
1493 1764 : if (ls->supported_scale)
1494 0 : scale = ls->supported_scale;
1495 :
1496 1764 : if (internal_gather_scatter_fn_supported_p (len_ifn, vectype,
1497 : memory_type,
1498 : off_vectype, scale,
1499 : elsvals))
1500 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, 1);
1501 1764 : else if (internal_gather_scatter_fn_supported_p (ifn, vectype,
1502 : memory_type,
1503 : off_vectype, scale,
1504 : elsvals)
1505 1764 : || memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
1506 567 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
1507 : scalar_mask);
1508 : else
1509 : {
1510 1197 : if (dump_enabled_p ())
1511 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1512 : "can't operate on partial vectors because"
1513 : " the target doesn't have an appropriate"
1514 : " gather load or scatter store instruction.\n");
1515 1197 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1516 : }
1517 : return;
1518 : }
1519 :
1520 294501 : if (memory_access_type != VMAT_CONTIGUOUS)
1521 : {
1522 : /* Element X of the data must come from iteration i * VF + X of the
1523 : scalar loop. We need more work to support other mappings. */
1524 26527 : if (dump_enabled_p ())
1525 733 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1526 : "can't operate on partial vectors because an"
1527 : " access isn't contiguous.\n");
1528 26527 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1529 26527 : return;
1530 : }
1531 :
1532 267974 : if (!VECTOR_MODE_P (vecmode))
1533 : {
1534 0 : if (dump_enabled_p ())
1535 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1536 : "can't operate on partial vectors when emulating"
1537 : " vector operations.\n");
1538 0 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1539 0 : return;
1540 : }
1541 :
1542 : /* We might load more scalars than we need for permuting SLP loads.
1543 : We checked in get_load_store_type that the extra elements
1544 : don't leak into a new vector. */
1545 358755 : auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
1546 : {
1547 90781 : unsigned int nvectors;
1548 90781 : if (can_div_away_from_zero_p (size, nunits, &nvectors))
1549 90781 : return nvectors;
1550 : gcc_unreachable ();
1551 : };
1552 :
1553 267974 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1554 267974 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1555 267974 : machine_mode mask_mode;
1556 267974 : machine_mode vmode;
1557 267974 : bool using_partial_vectors_p = false;
1558 267974 : if (get_len_load_store_mode
1559 267974 : (vecmode, is_load, nullptr, elsvals).exists (&vmode))
1560 : {
1561 0 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1562 0 : unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
1563 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
1564 0 : using_partial_vectors_p = true;
1565 : }
1566 358755 : else if (targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
1567 267974 : && can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
1568 : elsvals))
1569 : {
1570 90781 : nvectors = group_memory_nvectors (group_size * vf, nunits);
1571 90781 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype, scalar_mask);
1572 90781 : using_partial_vectors_p = true;
1573 : }
1574 :
1575 90781 : if (!using_partial_vectors_p)
1576 : {
1577 177193 : if (dump_enabled_p ())
1578 12036 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1579 : "can't operate on partial vectors because the"
1580 : " target doesn't have the appropriate partial"
1581 : " vectorization load or store.\n");
1582 177193 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
1583 : }
1584 : }
1585 :
1586 : /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1587 : form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1588 : that needs to be applied to all loads and stores in a vectorized loop.
1589 : Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1590 : otherwise return VEC_MASK & LOOP_MASK.
1591 :
1592 : MASK_TYPE is the type of both masks. If new statements are needed,
1593 : insert them before GSI. */
1594 :
1595 : tree
1596 1611 : prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
1597 : tree vec_mask, gimple_stmt_iterator *gsi)
1598 : {
1599 1611 : gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1600 1611 : if (!loop_mask)
1601 : return vec_mask;
1602 :
1603 141 : gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1604 :
1605 141 : if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
1606 : return vec_mask;
1607 :
1608 141 : tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1609 141 : gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1610 : vec_mask, loop_mask);
1611 :
1612 141 : gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1613 141 : return and_res;
1614 : }
1615 :
1616 : /* Determine whether we can use a gather load or scatter store to vectorize
1617 : strided load or store STMT_INFO by truncating the current offset to a
1618 : smaller width. We need to be able to construct an offset vector:
1619 :
1620 : { 0, X, X*2, X*3, ... }
1621 :
1622 : without loss of precision, where X is STMT_INFO's DR_STEP.
1623 :
1624 : Return true if this is possible, describing the gather load or scatter
1625 : store in GS_INFO. MASKED_P is true if the load or store is conditional.
1626 :
1627 : If we can use gather/scatter and ELSVALS is nonzero the supported
1628 : else values will be stored in the vector ELSVALS points to. */
1629 :
1630 : static bool
1631 64444 : vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
1632 : loop_vec_info loop_vinfo, bool masked_p,
1633 : gather_scatter_info *gs_info,
1634 : vec<int> *elsvals)
1635 : {
1636 64444 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1637 64444 : data_reference *dr = dr_info->dr;
1638 64444 : tree step = DR_STEP (dr);
1639 64444 : if (TREE_CODE (step) != INTEGER_CST)
1640 : {
1641 : /* ??? Perhaps we could use range information here? */
1642 29005 : if (dump_enabled_p ())
1643 205 : dump_printf_loc (MSG_NOTE, vect_location,
1644 : "cannot truncate variable step.\n");
1645 : return false;
1646 : }
1647 :
1648 : /* Get the number of bits in an element. */
1649 35439 : scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1650 35439 : unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1651 :
1652 : /* Set COUNT to the upper limit on the number of elements - 1.
1653 : Start with the maximum vectorization factor. */
1654 35439 : unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1655 :
1656 : /* Try lowering COUNT to the number of scalar latch iterations. */
1657 35439 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1658 35439 : widest_int max_iters;
1659 35439 : if (max_loop_iterations (loop, &max_iters)
1660 70148 : && max_iters < count)
1661 2116 : count = max_iters.to_shwi ();
1662 :
1663 : /* Try scales of 1 and the element size. */
1664 35439 : unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
1665 35439 : wi::overflow_type overflow = wi::OVF_NONE;
1666 106317 : for (int i = 0; i < 2; ++i)
1667 : {
1668 70878 : unsigned int scale = scales[i];
1669 70878 : widest_int factor;
1670 70878 : if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1671 0 : continue;
1672 :
1673 : /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1674 70878 : widest_int range = wi::mul (count, factor, SIGNED, &overflow);
1675 70878 : if (overflow)
1676 0 : continue;
1677 70878 : signop sign = range >= 0 ? UNSIGNED : SIGNED;
1678 70878 : unsigned int min_offset_bits = wi::min_precision (range, sign);
1679 :
1680 : /* Find the narrowest viable offset type. */
1681 70878 : unsigned int offset_bits = 1U << ceil_log2 (min_offset_bits);
1682 70878 : tree offset_type = build_nonstandard_integer_type (offset_bits,
1683 : sign == UNSIGNED);
1684 :
1685 : /* See whether the target supports the operation with an offset
1686 : no narrower than OFFSET_TYPE. */
1687 70878 : tree memory_type = TREE_TYPE (DR_REF (dr));
1688 70878 : tree tmp_offset_vectype;
1689 70878 : int tmp_scale;
1690 70878 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
1691 : vectype, memory_type, offset_type,
1692 : scale, &tmp_scale,
1693 : &gs_info->ifn, &gs_info->offset_vectype,
1694 : &tmp_offset_vectype, elsvals)
1695 70878 : || gs_info->ifn == IFN_LAST)
1696 70878 : continue;
1697 :
1698 0 : gs_info->decl = NULL_TREE;
1699 : /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1700 : but we don't need to store that here. */
1701 0 : gs_info->base = NULL_TREE;
1702 0 : gs_info->alias_ptr = build_int_cst
1703 0 : (reference_alias_ptr_type (DR_REF (dr)),
1704 0 : get_object_alignment (DR_REF (dr)));
1705 0 : gs_info->element_type = TREE_TYPE (vectype);
1706 0 : gs_info->offset = fold_convert (offset_type, step);
1707 0 : gs_info->scale = scale;
1708 0 : gs_info->memory_type = memory_type;
1709 0 : return true;
1710 70878 : }
1711 :
1712 35439 : if (overflow && dump_enabled_p ())
1713 0 : dump_printf_loc (MSG_NOTE, vect_location,
1714 : "truncating gather/scatter offset to %d bits"
1715 : " might change its value.\n", element_bits);
1716 :
1717 : return false;
1718 64444 : }
1719 :
1720 : /* Return true if we can use gather/scatter or strided internal functions
1721 : to vectorize STMT_INFO, which is a grouped or strided load or store
1722 : with multiple lanes and will be implemented by a type-punned access
1723 : of a vector with element size that matches the number of lanes.
1724 :
1725 : MASKED_P is true if load or store is conditional.
1726 : When returning true, fill in GS_INFO with the information required to
1727 : perform the operation. Also, store the punning type in PUNNED_VECTYPE.
1728 :
1729 : If successful and ELSVALS is nonzero the supported
1730 : else values will be stored in the vector ELSVALS points to. */
1731 :
1732 : static bool
1733 4658 : vect_use_grouped_gather (dr_vec_info *dr_info, tree vectype,
1734 : loop_vec_info loop_vinfo, bool masked_p,
1735 : unsigned int nelts,
1736 : gather_scatter_info *info, vec<int> *elsvals,
1737 : tree *pun_vectype)
1738 : {
1739 4658 : data_reference *dr = dr_info->dr;
1740 :
1741 : /* TODO: We can support nelts > BITS_PER_UNIT or non-power-of-two by
1742 : multiple gathers/scatter. */
1743 4658 : if (nelts > BITS_PER_UNIT || !pow2p_hwi (nelts))
1744 : return false;
1745 :
1746 : /* Pun the vectype with one of the same size but an element spanning
1747 : NELTS elements of VECTYPE.
1748 : The punned type of a V16QI with NELTS = 4 would be V4SI.
1749 : */
1750 4025 : tree tmp;
1751 4025 : unsigned int pieces;
1752 4025 : if (!can_div_trunc_p (TYPE_VECTOR_SUBPARTS (vectype), nelts, &pieces)
1753 4025 : || pieces <= 1)
1754 2009 : return false;
1755 :
1756 2016 : *pun_vectype = vector_vector_composition_type (vectype, pieces, &tmp, true);
1757 :
1758 2016 : if (!*pun_vectype || !VECTOR_TYPE_P (*pun_vectype))
1759 : return false;
1760 :
1761 1870 : internal_fn ifn;
1762 1870 : tree offset_vectype = *pun_vectype;
1763 :
1764 1263 : internal_fn strided_ifn = DR_IS_READ (dr)
1765 1870 : ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
1766 :
1767 : /* Check if we have a gather/scatter with the new type. We're just trying
1768 : with the type itself as offset for now. If not, check if we have a
1769 : strided load/store. These have fewer constraints (for example no offset
1770 : type must exist) so it is possible that even though a gather/scatter is
1771 : not available we still have a strided load/store. */
1772 1870 : bool ok = false;
1773 1870 : tree tmp_vectype;
1774 1870 : int tmp_scale;
1775 1870 : if (vect_gather_scatter_fn_p
1776 1870 : (loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
1777 1870 : TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
1778 : &offset_vectype, &tmp_vectype, elsvals))
1779 : ok = true;
1780 1870 : else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
1781 : elsvals))
1782 : {
1783 : /* Use gather/scatter IFNs, vect_get_strided_load_store_ops
1784 : will switch back to the strided variants. */
1785 0 : ifn = DR_IS_READ (dr) ? IFN_MASK_LEN_GATHER_LOAD :
1786 : IFN_MASK_LEN_SCATTER_STORE;
1787 0 : ok = true;
1788 : }
1789 :
1790 0 : if (ok)
1791 : {
1792 0 : info->ifn = ifn;
1793 0 : info->decl = NULL_TREE;
1794 0 : info->base = dr->ref;
1795 0 : info->alias_ptr = build_int_cst
1796 0 : (reference_alias_ptr_type (DR_REF (dr)),
1797 0 : get_object_alignment (DR_REF (dr)));
1798 0 : info->element_type = TREE_TYPE (*pun_vectype);
1799 0 : info->offset_vectype = offset_vectype;
1800 : /* No need to set the offset, vect_get_strided_load_store_ops
1801 : will do that. */
1802 0 : info->scale = 1;
1803 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
1804 0 : return true;
1805 : }
1806 :
1807 : return false;
1808 : }
1809 :
1810 :
1811 : /* Return true if we can use gather/scatter internal functions to
1812 : vectorize STMT_INFO, which is a grouped or strided load or store.
1813 : MASKED_P is true if load or store is conditional. When returning
1814 : true, fill in GS_INFO with the information required to perform the
1815 : operation.
1816 :
1817 : If we can use gather/scatter and ELSVALS is nonzero the supported
1818 : else values will be stored in the vector ELSVALS points to. */
1819 :
1820 : static bool
1821 64444 : vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
1822 : loop_vec_info loop_vinfo, bool masked_p,
1823 : gather_scatter_info *gs_info,
1824 : vec<int> *elsvals,
1825 : unsigned int group_size,
1826 : bool single_element_p)
1827 : {
1828 64444 : if (!vect_check_gather_scatter (stmt_info, vectype,
1829 : loop_vinfo, gs_info, elsvals)
1830 64444 : || gs_info->ifn == IFN_LAST)
1831 : {
1832 64444 : if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
1833 : masked_p, gs_info, elsvals))
1834 : return false;
1835 : }
1836 :
1837 0 : if (!single_element_p
1838 0 : && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
1839 : gs_info->scale,
1840 : group_size))
1841 : return false;
1842 :
1843 0 : if (dump_enabled_p ())
1844 0 : dump_printf_loc (MSG_NOTE, vect_location,
1845 : "using gather/scatter for strided/grouped access,"
1846 : " scale = %d\n", gs_info->scale);
1847 :
1848 : return true;
1849 : }
1850 :
1851 : /* STMT_INFO is a non-strided load or store, meaning that it accesses
1852 : elements with a known constant step. Return -1 if that step
1853 : is negative, 0 if it is zero, and 1 if it is greater than zero. */
1854 :
1855 : int
1856 1519545 : compare_step_with_zero (vec_info *vinfo, stmt_vec_info stmt_info)
1857 : {
1858 1519545 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1859 1519545 : return tree_int_cst_compare (vect_dr_behavior (vinfo, dr_info)->step,
1860 1519545 : size_zero_node);
1861 : }
1862 :
1863 : /* If the target supports a permute mask that reverses the elements in
1864 : a vector of type VECTYPE, return that mask, otherwise return null. */
1865 :
1866 : tree
1867 9205 : perm_mask_for_reverse (tree vectype)
1868 : {
1869 9205 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1870 :
1871 : /* The encoding has a single stepped pattern. */
1872 9205 : vec_perm_builder sel (nunits, 1, 3);
1873 46025 : for (int i = 0; i < 3; ++i)
1874 27615 : sel.quick_push (nunits - 1 - i);
1875 :
1876 9205 : vec_perm_indices indices (sel, 1, nunits);
1877 9205 : if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
1878 : indices))
1879 : return NULL_TREE;
1880 8027 : return vect_gen_perm_mask_checked (vectype, indices);
1881 9205 : }
1882 :
1883 : /* A subroutine of get_load_store_type, with a subset of the same
1884 : arguments. Handle the case where STMT_INFO is a load or store that
1885 : accesses consecutive elements with a negative step. Sets *POFFSET
1886 : to the offset to be applied to the DR for the first access. */
1887 :
1888 : static vect_memory_access_type
1889 12338 : get_negative_load_store_type (vec_info *vinfo,
1890 : stmt_vec_info stmt_info, tree vectype,
1891 : vec_load_store_type vls_type,
1892 : unsigned int ncopies, poly_int64 *poffset)
1893 : {
1894 12338 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
1895 12338 : dr_alignment_support alignment_support_scheme;
1896 :
1897 12338 : if (ncopies > 1)
1898 : {
1899 0 : if (dump_enabled_p ())
1900 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1901 : "multiple types with negative step.\n");
1902 : return VMAT_ELEMENTWISE;
1903 : }
1904 :
1905 : /* For backward running DRs the first access in vectype actually is
1906 : N-1 elements before the address of the DR. */
1907 12338 : *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
1908 12338 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1909 :
1910 12338 : int misalignment = dr_misalignment (dr_info, vectype, *poffset);
1911 12338 : alignment_support_scheme
1912 12338 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
1913 12338 : if (alignment_support_scheme != dr_aligned
1914 12338 : && alignment_support_scheme != dr_unaligned_supported)
1915 : {
1916 4516 : if (dump_enabled_p ())
1917 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1918 : "negative step but alignment required.\n");
1919 4516 : *poffset = 0;
1920 4516 : return VMAT_ELEMENTWISE;
1921 : }
1922 :
1923 7822 : if (vls_type == VLS_STORE_INVARIANT)
1924 : {
1925 1197 : if (dump_enabled_p ())
1926 21 : dump_printf_loc (MSG_NOTE, vect_location,
1927 : "negative step with invariant source;"
1928 : " no permute needed.\n");
1929 : return VMAT_CONTIGUOUS_DOWN;
1930 : }
1931 :
1932 6625 : if (!perm_mask_for_reverse (vectype))
1933 : {
1934 1178 : if (dump_enabled_p ())
1935 52 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1936 : "negative step and reversing not supported.\n");
1937 1178 : *poffset = 0;
1938 1178 : return VMAT_ELEMENTWISE;
1939 : }
1940 :
1941 : return VMAT_CONTIGUOUS_REVERSE;
1942 : }
1943 :
1944 : /* STMT_INFO is either a masked or unconditional store. Return the value
1945 : being stored. */
1946 :
1947 : tree
1948 0 : vect_get_store_rhs (stmt_vec_info stmt_info)
1949 : {
1950 0 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
1951 : {
1952 0 : gcc_assert (gimple_assign_single_p (assign));
1953 0 : return gimple_assign_rhs1 (assign);
1954 : }
1955 0 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
1956 : {
1957 0 : internal_fn ifn = gimple_call_internal_fn (call);
1958 0 : int index = internal_fn_stored_value_index (ifn);
1959 0 : gcc_assert (index >= 0);
1960 0 : return gimple_call_arg (call, index);
1961 : }
1962 0 : gcc_unreachable ();
1963 : }
1964 :
1965 : /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1966 :
1967 : This function returns a vector type which can be composed with NELTS pieces,
1968 : whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1969 : same vector size as the return vector. It checks target whether supports
1970 : pieces-size vector mode for construction firstly, if target fails to, check
1971 : pieces-size scalar mode for construction further. It returns NULL_TREE if
1972 : fails to find the available composition. If the caller only wants scalar
1973 : pieces where PTYPE e.g. is a possible gather/scatter element type
1974 : SCALAR_PTYPE_ONLY must be true.
1975 :
1976 : For example, for (vtype=V16QI, nelts=4), we can probably get:
1977 : - V16QI with PTYPE V4QI.
1978 : - V4SI with PTYPE SI.
1979 : - NULL_TREE. */
1980 :
1981 : static tree
1982 13421 : vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype,
1983 : bool scalar_ptype_only)
1984 : {
1985 13421 : gcc_assert (VECTOR_TYPE_P (vtype));
1986 13421 : gcc_assert (known_gt (nelts, 0U));
1987 :
1988 13421 : machine_mode vmode = TYPE_MODE (vtype);
1989 13421 : if (!VECTOR_MODE_P (vmode))
1990 : return NULL_TREE;
1991 :
1992 : /* When we are asked to compose the vector from its components let
1993 : that happen directly. */
1994 13421 : if (known_eq (TYPE_VECTOR_SUBPARTS (vtype), nelts))
1995 : {
1996 6759 : *ptype = TREE_TYPE (vtype);
1997 6759 : return vtype;
1998 : }
1999 :
2000 13324 : poly_uint64 vbsize = GET_MODE_BITSIZE (vmode);
2001 6662 : unsigned int pbsize;
2002 6662 : if (constant_multiple_p (vbsize, nelts, &pbsize))
2003 : {
2004 : /* First check if vec_init optab supports construction from
2005 : vector pieces directly. */
2006 6662 : scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vtype));
2007 13324 : poly_uint64 inelts = pbsize / GET_MODE_BITSIZE (elmode);
2008 6662 : machine_mode rmode;
2009 6662 : if (!scalar_ptype_only
2010 8192 : && related_vector_mode (vmode, elmode, inelts).exists (&rmode)
2011 10846 : && (convert_optab_handler (vec_init_optab, vmode, rmode)
2012 : != CODE_FOR_nothing))
2013 : {
2014 3546 : *ptype = build_vector_type (TREE_TYPE (vtype), inelts);
2015 3546 : return vtype;
2016 : }
2017 :
2018 : /* Otherwise check if exists an integer type of the same piece size and
2019 : if vec_init optab supports construction from it directly. */
2020 3116 : if (int_mode_for_size (pbsize, 0).exists (&elmode)
2021 3116 : && related_vector_mode (vmode, elmode, nelts).exists (&rmode))
2022 : {
2023 2932 : if (scalar_ptype_only
2024 2932 : || convert_optab_handler (vec_init_optab, rmode, elmode)
2025 : != CODE_FOR_nothing)
2026 : {
2027 2932 : *ptype = build_nonstandard_integer_type (pbsize, 1);
2028 2932 : return build_vector_type (*ptype, nelts);
2029 : }
2030 : }
2031 : }
2032 :
2033 : return NULL_TREE;
2034 : }
2035 :
2036 : /* Check if the load permutation of NODE only refers to a consecutive
2037 : subset of the group indices. We also require the length of the
2038 : permutation to be a power of two.
2039 : Such load permutations can be elided in strided access schemes as
2040 : we can "jump over" the gap they leave. */
2041 :
2042 : static bool
2043 45517 : has_consecutive_load_permutation (slp_tree node)
2044 : {
2045 45517 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
2046 45517 : if (!perm.exists ()
2047 2183 : || perm.length () <= 1
2048 46026 : || !pow2p_hwi (perm.length ()))
2049 : return false;
2050 :
2051 489 : return vect_load_perm_consecutive_p (node);
2052 : }
2053 :
2054 :
2055 : /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
2056 : if there is a memory access type that the vectorized form can use,
2057 : storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2058 : or scatters, fill in GS_INFO accordingly. In addition
2059 : *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2060 : the target does not support the alignment scheme. *MISALIGNMENT
2061 : is set according to the alignment of the access (including
2062 : DR_MISALIGNMENT_UNKNOWN when it is unknown).
2063 :
2064 : MASKED_P is true if the statement is conditional on a vectorized mask.
2065 : VECTYPE is the vector type that the vectorized statements will use.
2066 :
2067 : If ELSVALS is nonzero the supported else values will be stored in the
2068 : vector ELSVALS points to. */
2069 :
2070 : static bool
2071 1404458 : get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
2072 : tree vectype, slp_tree slp_node,
2073 : bool masked_p, vec_load_store_type vls_type,
2074 : vect_load_store_data *ls)
2075 : {
2076 1404458 : vect_memory_access_type *memory_access_type = &ls->memory_access_type;
2077 1404458 : poly_int64 *poffset = &ls->poffset;
2078 1404458 : dr_alignment_support *alignment_support_scheme
2079 : = &ls->alignment_support_scheme;
2080 1404458 : int *misalignment = &ls->misalignment;
2081 1404458 : internal_fn *lanes_ifn = &ls->lanes_ifn;
2082 1404458 : vec<int> *elsvals = &ls->elsvals;
2083 1404458 : tree *ls_type = &ls->ls_type;
2084 1404458 : bool *slp_perm = &ls->slp_perm;
2085 1404458 : unsigned *n_perms = &ls->n_perms;
2086 1404458 : unsigned *n_loads = &ls->n_loads;
2087 1404458 : tree *supported_offset_vectype = &ls->supported_offset_vectype;
2088 1404458 : int *supported_scale = &ls->supported_scale;
2089 1404458 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
2090 1404458 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2091 1404458 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2092 1404458 : stmt_vec_info first_stmt_info;
2093 1404458 : unsigned int group_size;
2094 1404458 : unsigned HOST_WIDE_INT gap;
2095 1404458 : bool single_element_p;
2096 1404458 : poly_int64 neg_ldst_offset = 0;
2097 :
2098 1404458 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2099 1404458 : *poffset = 0;
2100 1404458 : *ls_type = NULL_TREE;
2101 1404458 : *slp_perm = false;
2102 1404458 : *n_perms = -1U;
2103 1404458 : *n_loads = -1U;
2104 1404458 : ls->subchain_p = false;
2105 :
2106 1404458 : bool perm_ok = true;
2107 1404458 : poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
2108 :
2109 1404458 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2110 80701 : perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL,
2111 80701 : vf, true, n_perms, n_loads);
2112 :
2113 1404458 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2114 : {
2115 898980 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
2116 898980 : group_size = DR_GROUP_SIZE (first_stmt_info);
2117 898980 : gap = DR_GROUP_GAP (first_stmt_info);
2118 898980 : single_element_p = (stmt_info == first_stmt_info
2119 898980 : && !DR_GROUP_NEXT_ELEMENT (stmt_info));
2120 : }
2121 : else
2122 : {
2123 : first_stmt_info = stmt_info;
2124 : group_size = 1;
2125 : gap = 0;
2126 : single_element_p = true;
2127 : }
2128 1404458 : dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
2129 :
2130 : /* True if the vectorized statements would access beyond the last
2131 : statement in the group. */
2132 1404458 : bool overrun_p = false;
2133 :
2134 : /* True if we can cope with such overrun by peeling for gaps, so that
2135 : there is at least one final scalar iteration after the vector loop. */
2136 2808916 : bool can_overrun_p = (!masked_p
2137 1404458 : && vls_type == VLS_LOAD
2138 571135 : && loop_vinfo
2139 1833722 : && !loop->inner);
2140 :
2141 : /* There can only be a gap at the end of the group if the stride is
2142 : known at compile time. */
2143 1404458 : gcc_assert (!STMT_VINFO_STRIDED_P (first_stmt_info) || gap == 0);
2144 :
2145 : /* For SLP vectorization we directly vectorize a subchain
2146 : without permutation. */
2147 1404458 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2148 1323757 : first_dr_info = STMT_VINFO_DR_INFO (SLP_TREE_SCALAR_STMTS (slp_node)[0]);
2149 :
2150 1404458 : if (STMT_VINFO_STRIDED_P (first_stmt_info))
2151 : {
2152 : /* Try to use consecutive accesses of as many elements as possible,
2153 : separated by the stride, until we have a complete vector.
2154 : Fall back to scalar accesses if that isn't possible. */
2155 45517 : *memory_access_type = VMAT_STRIDED_SLP;
2156 :
2157 : /* If the load permutation is consecutive we can reduce the group to
2158 : the elements the permutation accesses. Then we release the
2159 : permutation. */
2160 45517 : if (has_consecutive_load_permutation (slp_node))
2161 : {
2162 51 : ls->subchain_p = true;
2163 51 : group_size = SLP_TREE_LANES (slp_node);
2164 51 : SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
2165 : }
2166 : }
2167 1358941 : else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2168 : {
2169 10854 : slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
2170 10854 : tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
2171 10854 : int scale = SLP_TREE_GS_SCALE (slp_node);
2172 10854 : tree memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
2173 10854 : tree tem;
2174 10854 : if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
2175 : masked_p, vectype, memory_type,
2176 : offset_vectype, scale, supported_scale,
2177 : &ls->gs.ifn, &tem,
2178 : supported_offset_vectype, elsvals))
2179 : {
2180 0 : if (dump_enabled_p ())
2181 : {
2182 0 : dump_printf_loc (MSG_NOTE, vect_location,
2183 : "gather/scatter with required "
2184 : "offset type "
2185 : "%T and offset scale %d.\n",
2186 : offset_vectype, scale);
2187 0 : if (*supported_offset_vectype)
2188 0 : dump_printf_loc (MSG_NOTE, vect_location,
2189 : " target supports offset type %T.\n",
2190 : *supported_offset_vectype);
2191 0 : if (*supported_scale)
2192 0 : dump_printf_loc (MSG_NOTE, vect_location,
2193 : " target supports offset scale %d.\n",
2194 : *supported_scale);
2195 : }
2196 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2197 : }
2198 10854 : else if (vls_type == VLS_LOAD
2199 10854 : ? (targetm.vectorize.builtin_gather
2200 9263 : && (ls->gs.decl
2201 9263 : = targetm.vectorize.builtin_gather (vectype,
2202 9263 : TREE_TYPE
2203 : (offset_vectype),
2204 : scale)))
2205 1591 : : (targetm.vectorize.builtin_scatter
2206 1591 : && (ls->gs.decl
2207 1591 : = targetm.vectorize.builtin_scatter (vectype,
2208 1591 : TREE_TYPE
2209 : (offset_vectype),
2210 : scale))))
2211 575 : *memory_access_type = VMAT_GATHER_SCATTER_LEGACY;
2212 : else
2213 : {
2214 : /* GATHER_SCATTER_EMULATED_P. */
2215 10279 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
2216 10279 : || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
2217 10279 : || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
2218 10279 : || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
2219 13189 : TYPE_VECTOR_SUBPARTS (vectype)))
2220 : {
2221 2910 : if (dump_enabled_p ())
2222 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2223 : "unsupported vector types for emulated "
2224 : "gather.\n");
2225 2910 : return false;
2226 : }
2227 7369 : *memory_access_type = VMAT_GATHER_SCATTER_EMULATED;
2228 : }
2229 : }
2230 : else
2231 : {
2232 1348087 : int cmp = compare_step_with_zero (vinfo, stmt_info);
2233 1348087 : if (cmp < 0)
2234 : {
2235 12516 : if (single_element_p)
2236 : /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2237 : only correct for single element "interleaving" SLP. */
2238 12338 : *memory_access_type = get_negative_load_store_type
2239 12338 : (vinfo, stmt_info, vectype, vls_type, 1,
2240 : &neg_ldst_offset);
2241 : else
2242 : /* We can fall back to VMAT_STRIDED_SLP since that does
2243 : not care whether the stride between the group instances
2244 : is positive or negative. */
2245 178 : *memory_access_type = VMAT_STRIDED_SLP;
2246 : }
2247 1335571 : else if (cmp == 0 && loop_vinfo)
2248 : {
2249 3418 : gcc_assert (vls_type == VLS_LOAD);
2250 3418 : *memory_access_type = VMAT_INVARIANT;
2251 : }
2252 : /* Try using LOAD/STORE_LANES. */
2253 1332153 : else if (slp_node->ldst_lanes
2254 1332153 : && (*lanes_ifn
2255 0 : = (vls_type == VLS_LOAD
2256 0 : ? vect_load_lanes_supported (vectype, group_size,
2257 : masked_p, elsvals)
2258 0 : : vect_store_lanes_supported (vectype, group_size,
2259 : masked_p))) != IFN_LAST)
2260 0 : *memory_access_type = VMAT_LOAD_STORE_LANES;
2261 1332153 : else if (!loop_vinfo && slp_node->avoid_stlf_fail)
2262 : {
2263 70 : *memory_access_type = VMAT_ELEMENTWISE;
2264 70 : if (dump_enabled_p ())
2265 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2266 : "using element-wise load to avoid disrupting "
2267 : "cross iteration store-to-load forwarding\n");
2268 : }
2269 : else
2270 1332083 : *memory_access_type = VMAT_CONTIGUOUS;
2271 :
2272 : /* If this is single-element interleaving with an element
2273 : distance that leaves unused vector loads around fall back
2274 : to elementwise access if possible - we otherwise least
2275 : create very sub-optimal code in that case (and
2276 : blow up memory, see PR65518). */
2277 1348087 : if (loop_vinfo
2278 1348087 : && single_element_p
2279 485875 : && (*memory_access_type == VMAT_CONTIGUOUS
2280 15756 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2281 1833962 : && maybe_gt (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
2282 : {
2283 17861 : *memory_access_type = VMAT_ELEMENTWISE;
2284 17861 : if (dump_enabled_p ())
2285 198 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2286 : "single-element interleaving not supported "
2287 : "for not adjacent vector loads, using "
2288 : "elementwise access\n");
2289 : }
2290 :
2291 : /* Also fall back to elementwise access in case we did not lower a
2292 : permutation and cannot code generate it. */
2293 1348087 : if (loop_vinfo
2294 540587 : && *memory_access_type != VMAT_ELEMENTWISE
2295 517032 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2296 1376883 : && !perm_ok)
2297 : {
2298 2087 : *memory_access_type = VMAT_ELEMENTWISE;
2299 2087 : if (dump_enabled_p ())
2300 248 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2301 : "permutation not supported, using elementwise "
2302 : "access\n");
2303 : }
2304 :
2305 540587 : overrun_p = (loop_vinfo && gap != 0
2306 1391286 : && *memory_access_type != VMAT_ELEMENTWISE);
2307 1348087 : if (overrun_p && vls_type != VLS_LOAD)
2308 : {
2309 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2310 : "Grouped store with gaps requires"
2311 : " non-consecutive accesses\n");
2312 0 : return false;
2313 : }
2314 :
2315 1348087 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (first_dr_info);
2316 1348087 : poly_int64 off = 0;
2317 1348087 : if (*memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2318 5288 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2319 :
2320 : /* An overrun is fine if the trailing elements are smaller
2321 : than the alignment boundary B. Every vector access will
2322 : be a multiple of B and so we are guaranteed to access a
2323 : non-gap element in the same B-sized block. */
2324 1348087 : if (overrun_p
2325 1348087 : && gap < (vect_known_alignment_in_bytes (first_dr_info,
2326 23182 : vectype, off) / dr_size))
2327 : overrun_p = false;
2328 :
2329 : /* When we have a contiguous access across loop iterations
2330 : but the access in the loop doesn't cover the full vector
2331 : we can end up with no gap recorded but still excess
2332 : elements accessed, see PR103116. Make sure we peel for
2333 : gaps if necessary and sufficient and give up if not.
2334 :
2335 : If there is a combination of the access not covering the full
2336 : vector and a gap recorded then we may need to peel twice. */
2337 1348087 : bool large_vector_overrun_p = false;
2338 1348087 : if (loop_vinfo
2339 540587 : && (*memory_access_type == VMAT_CONTIGUOUS
2340 35718 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2341 510157 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2342 1374415 : && !multiple_p (group_size * LOOP_VINFO_VECT_FACTOR (loop_vinfo),
2343 : nunits))
2344 : large_vector_overrun_p = overrun_p = true;
2345 :
2346 : /* If the gap splits the vector in half and the target
2347 : can do half-vector operations avoid the epilogue peeling
2348 : by simply loading half of the vector only. Usually
2349 : the construction with an upper zero half will be elided. */
2350 1348087 : dr_alignment_support alss;
2351 1348087 : int misalign = dr_misalignment (first_dr_info, vectype, off);
2352 1348087 : tree half_vtype;
2353 1348087 : poly_uint64 remain;
2354 1348087 : unsigned HOST_WIDE_INT tem, num;
2355 1348087 : if (overrun_p
2356 1348087 : && !masked_p
2357 17665 : && *memory_access_type != VMAT_LOAD_STORE_LANES
2358 17665 : && (((alss = vect_supportable_dr_alignment (vinfo, first_dr_info,
2359 : vectype, misalign)))
2360 : == dr_aligned
2361 15125 : || alss == dr_unaligned_supported)
2362 9983 : && can_div_trunc_p (group_size
2363 9983 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2364 : nunits, &tem, &remain)
2365 1358070 : && (known_eq (remain, 0u)
2366 7476 : || (known_ne (remain, 0u)
2367 5784 : && constant_multiple_p (nunits, remain, &num)
2368 1348087 : && (vector_vector_composition_type (vectype, num, &half_vtype)
2369 : != NULL_TREE))))
2370 : overrun_p = false;
2371 :
2372 1348087 : if (overrun_p && !can_overrun_p)
2373 : {
2374 6 : if (dump_enabled_p ())
2375 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2376 : "Peeling for outer loop is not supported\n");
2377 : return false;
2378 : }
2379 :
2380 : /* Peeling for gaps assumes that a single scalar iteration
2381 : is enough to make sure the last vector iteration doesn't
2382 : access excess elements. */
2383 1348081 : if (overrun_p
2384 1348081 : && (!can_div_trunc_p (group_size
2385 9368 : * LOOP_VINFO_VECT_FACTOR (loop_vinfo) - gap,
2386 : nunits, &tem, &remain)
2387 9368 : || maybe_lt (remain + group_size, nunits)))
2388 : {
2389 : /* But peeling a single scalar iteration is enough if
2390 : we can use the next power-of-two sized partial
2391 : access and that is sufficiently small to be covered
2392 : by the single scalar iteration. */
2393 16 : unsigned HOST_WIDE_INT cnunits, cvf, cremain, cpart_size;
2394 16 : if (masked_p
2395 16 : || !nunits.is_constant (&cnunits)
2396 16 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&cvf)
2397 16 : || (((cremain = (group_size * cvf - gap) % cnunits), true)
2398 16 : && ((cpart_size = (1 << ceil_log2 (cremain))), true)
2399 16 : && (cremain + group_size < cpart_size
2400 16 : || (vector_vector_composition_type (vectype,
2401 13 : cnunits / cpart_size,
2402 : &half_vtype)
2403 : == NULL_TREE))))
2404 : {
2405 : /* If all fails we can still resort to niter masking unless
2406 : the vectors used are too big, so enforce the use of
2407 : partial vectors. */
2408 3 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
2409 3 : && !large_vector_overrun_p)
2410 : {
2411 0 : if (dump_enabled_p ())
2412 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2413 : "peeling for gaps insufficient for "
2414 : "access unless using partial "
2415 : "vectors\n");
2416 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2417 : }
2418 : else
2419 : {
2420 3 : if (dump_enabled_p ())
2421 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2422 : "peeling for gaps insufficient for "
2423 : "access\n");
2424 9 : return false;
2425 : }
2426 : }
2427 13 : else if (large_vector_overrun_p)
2428 : {
2429 13 : if (dump_enabled_p ())
2430 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2431 : "can't operate on partial vectors because "
2432 : "only unmasked loads handle access "
2433 : "shortening required because of gaps at "
2434 : "the end of the access\n");
2435 13 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
2436 : }
2437 : }
2438 : }
2439 :
2440 : /* As a last resort, trying using a gather load or scatter store.
2441 :
2442 : ??? Although the code can handle all group sizes correctly,
2443 : it probably isn't a win to use separate strided accesses based
2444 : on nearby locations. Or, even if it's a win over scalar code,
2445 : it might not be a win over vectorizing at a lower VF, if that
2446 : allows us to use contiguous accesses. */
2447 1401539 : vect_memory_access_type grouped_gather_fallback = VMAT_UNINITIALIZED;
2448 1401539 : if (loop_vinfo
2449 594039 : && (*memory_access_type == VMAT_ELEMENTWISE
2450 594039 : || *memory_access_type == VMAT_STRIDED_SLP))
2451 : {
2452 71332 : gather_scatter_info gs_info;
2453 71332 : tree tem;
2454 71332 : if (SLP_TREE_LANES (slp_node) == 1
2455 66480 : && (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2456 21656 : || single_element_p)
2457 135776 : && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
2458 : masked_p, &gs_info, elsvals,
2459 : group_size, single_element_p))
2460 : {
2461 : /* vect_use_strided_gather_scatters_p does not save the actually
2462 : supported scale and offset type so do that here.
2463 : We need it later in check_load_store_for_partial_vectors
2464 : where we only check if the given internal function is supported
2465 : (to choose whether to use the IFN, LEGACY, or EMULATED flavor
2466 : of gather/scatter) and don't re-do the full analysis. */
2467 0 : tree tmp;
2468 0 : gcc_assert (vect_gather_scatter_fn_p
2469 : (loop_vinfo, vls_type == VLS_LOAD, masked_p, vectype,
2470 : gs_info.memory_type, TREE_TYPE (gs_info.offset),
2471 : gs_info.scale, supported_scale, &gs_info.ifn,
2472 : &tmp, supported_offset_vectype, elsvals));
2473 :
2474 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2475 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2476 0 : ls->gs.ifn = gs_info.ifn;
2477 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2478 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2479 : }
2480 71332 : else if (SLP_TREE_LANES (slp_node) > 1
2481 : && !masked_p
2482 4852 : && !single_element_p
2483 75990 : && vect_use_grouped_gather (STMT_VINFO_DR_INFO (stmt_info),
2484 : vectype, loop_vinfo,
2485 : masked_p, group_size,
2486 : &gs_info, elsvals, &tem))
2487 : {
2488 0 : SLP_TREE_GS_SCALE (slp_node) = gs_info.scale;
2489 0 : SLP_TREE_GS_BASE (slp_node) = error_mark_node;
2490 0 : grouped_gather_fallback = *memory_access_type;
2491 0 : *memory_access_type = VMAT_GATHER_SCATTER_IFN;
2492 0 : ls->gs.ifn = gs_info.ifn;
2493 0 : vectype = *ls_type = tem;
2494 0 : ls->strided_offset_vectype = gs_info.offset_vectype;
2495 : }
2496 : }
2497 :
2498 1401539 : if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
2499 1401539 : || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
2500 6481 : *poffset = neg_ldst_offset;
2501 :
2502 1401539 : if (*memory_access_type == VMAT_ELEMENTWISE
2503 1375827 : || *memory_access_type == VMAT_GATHER_SCATTER_LEGACY
2504 1375252 : || *memory_access_type == VMAT_STRIDED_SLP
2505 1329562 : || *memory_access_type == VMAT_INVARIANT)
2506 : {
2507 75395 : *alignment_support_scheme = dr_unaligned_supported;
2508 75395 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2509 : }
2510 : else
2511 : {
2512 1326144 : if (mat_gather_scatter_p (*memory_access_type)
2513 : && !first_dr_info)
2514 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2515 : else
2516 1326144 : *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
2517 1326144 : *alignment_support_scheme
2518 1326144 : = vect_supportable_dr_alignment
2519 1326144 : (vinfo, first_dr_info, vectype, *misalignment,
2520 : mat_gather_scatter_p (*memory_access_type));
2521 1326144 : if (grouped_gather_fallback != VMAT_UNINITIALIZED
2522 0 : && *alignment_support_scheme != dr_aligned
2523 0 : && *alignment_support_scheme != dr_unaligned_supported)
2524 : {
2525 : /* No supportable alignment for a grouped gather, fall back to the
2526 : original memory access type. Even though VMAT_STRIDED_SLP might
2527 : also try aligned vector loads it can still choose vector
2528 : construction from scalars. */
2529 0 : *memory_access_type = grouped_gather_fallback;
2530 0 : *alignment_support_scheme = dr_unaligned_supported;
2531 0 : *misalignment = DR_MISALIGNMENT_UNKNOWN;
2532 : }
2533 : }
2534 :
2535 1401539 : if (overrun_p)
2536 : {
2537 9365 : gcc_assert (can_overrun_p);
2538 9365 : if (dump_enabled_p ())
2539 511 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2540 : "Data access with gaps requires scalar "
2541 : "epilogue loop\n");
2542 9365 : LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2543 : }
2544 :
2545 1401539 : if ((*memory_access_type == VMAT_ELEMENTWISE
2546 1401539 : || *memory_access_type == VMAT_STRIDED_SLP)
2547 : && !nunits.is_constant ())
2548 : {
2549 : if (dump_enabled_p ())
2550 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2551 : "Not using elementwise accesses due to variable "
2552 : "vectorization factor.\n");
2553 : return false;
2554 : }
2555 :
2556 : /* Checks if all scalar iterations are known to be inbounds. */
2557 1401539 : bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
2558 :
2559 : /* Check if we support the operation if early breaks are needed. Here we
2560 : must ensure that we don't access any more than the scalar code would
2561 : have. A masked operation would ensure this, so for these load types
2562 : force masking. */
2563 1401539 : if (loop_vinfo
2564 594039 : && dr_safe_speculative_read_required (stmt_info)
2565 1594987 : && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2566 : {
2567 193448 : if (mat_gather_scatter_p (*memory_access_type)
2568 193448 : || *memory_access_type == VMAT_STRIDED_SLP)
2569 : {
2570 9484 : if (dump_enabled_p ())
2571 8 : dump_printf_loc (MSG_NOTE, vect_location,
2572 : "early break not supported: cannot peel for "
2573 : "alignment. With non-contiguous memory vectorization"
2574 : " could read out of bounds at %G ",
2575 : STMT_VINFO_STMT (stmt_info));
2576 9484 : if (inbounds)
2577 0 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2578 : else
2579 : return false;
2580 : }
2581 : /* Block-level alignment: Even though individual accesses of
2582 : VMAT_ELEMENTWISE type do not cause alignment problems, loading the
2583 : whole vector's worth of values in a speculative early-break context
2584 : might cross a page boundary. Set the alignment scheme to `dr_aligned'
2585 : here in order to force checking of whether such accesses meet
2586 : alignment criteria. */
2587 183964 : else if (*memory_access_type == VMAT_ELEMENTWISE && !inbounds)
2588 15011 : *alignment_support_scheme = dr_aligned;
2589 : }
2590 :
2591 : /* If this DR needs alignment for correctness, we must ensure the target
2592 : alignment is a constant power-of-two multiple of the amount read per
2593 : vector iteration or force masking. */
2594 1392055 : if (dr_safe_speculative_read_required (stmt_info)
2595 1392055 : && (*alignment_support_scheme == dr_aligned
2596 110653 : && !mat_gather_scatter_p (*memory_access_type)))
2597 : {
2598 : /* We can only peel for loops, of course. */
2599 110653 : gcc_checking_assert (loop_vinfo);
2600 :
2601 110653 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2602 110653 : poly_uint64 read_amount
2603 110653 : = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
2604 110653 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2605 110653 : read_amount *= group_size;
2606 :
2607 110653 : auto target_alignment
2608 110653 : = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info));
2609 110653 : if (!multiple_p (target_alignment, read_amount))
2610 : {
2611 12712 : if (dump_enabled_p ())
2612 : {
2613 28 : dump_printf_loc (MSG_NOTE, vect_location,
2614 : "desired alignment not met, target was ");
2615 28 : dump_dec (MSG_NOTE, target_alignment);
2616 28 : dump_printf (MSG_NOTE, " previously, but read amount is ");
2617 28 : dump_dec (MSG_NOTE, read_amount);
2618 28 : dump_printf (MSG_NOTE, " at %G.\n", STMT_VINFO_STMT (stmt_info));
2619 : }
2620 14954 : return false;
2621 : }
2622 :
2623 : /* When using a group access the first element may be aligned but the
2624 : subsequent loads may not be. For LOAD_LANES since the loads are based
2625 : on the first DR then all loads in the group are aligned. For
2626 : non-LOAD_LANES this is not the case. In particular a load + blend when
2627 : there are gaps can have the non first loads issued unaligned, even
2628 : partially overlapping the memory of the first load in order to simplify
2629 : the blend. This is what the x86_64 backend does for instance. As
2630 : such only the first load in the group is aligned, the rest are not.
2631 : Because of this the permutes may break the alignment requirements that
2632 : have been set, and as such we should for now, reject them. */
2633 97941 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
2634 : {
2635 2242 : if (dump_enabled_p ())
2636 87 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2637 : "loads with load permutations not supported for "
2638 : "speculative early break loads for %G",
2639 : STMT_VINFO_STMT (stmt_info));
2640 : return false;
2641 : }
2642 :
2643 : /* Reject vectorization if we know the read mount per vector iteration
2644 : exceeds the min page size. */
2645 95699 : if (known_gt (read_amount, (unsigned) param_min_pagesize))
2646 : {
2647 0 : if (dump_enabled_p ())
2648 : {
2649 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2650 : "alignment required for correctness (");
2651 0 : dump_dec (MSG_MISSED_OPTIMIZATION, read_amount);
2652 0 : dump_printf (MSG_NOTE, ") may exceed page size.\n");
2653 : }
2654 : return false;
2655 : }
2656 :
2657 95699 : if (!vf.is_constant ())
2658 : {
2659 : /* For VLA modes, we need a runtime check to ensure any speculative
2660 : read amount does not exceed the page size. Here we record the max
2661 : possible read amount for the check. */
2662 : if (maybe_gt (read_amount,
2663 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo)))
2664 : LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = read_amount;
2665 :
2666 : /* For VLA modes, we must use partial vectors. */
2667 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2668 : }
2669 : }
2670 :
2671 1377101 : if (*alignment_support_scheme == dr_unaligned_unsupported)
2672 : {
2673 69789 : if (dump_enabled_p ())
2674 256 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2675 : "unsupported unaligned access\n");
2676 : return false;
2677 : }
2678 :
2679 : /* FIXME: At the moment the cost model seems to underestimate the
2680 : cost of using elementwise accesses. This check preserves the
2681 : traditional behavior until that can be fixed. */
2682 1307312 : if (*memory_access_type == VMAT_ELEMENTWISE
2683 14998 : && !STMT_VINFO_STRIDED_P (first_stmt_info)
2684 1322310 : && !(STMT_VINFO_GROUPED_ACCESS (stmt_info)
2685 9727 : && single_element_p
2686 9080 : && !pow2p_hwi (group_size)))
2687 : {
2688 9291 : if (dump_enabled_p ())
2689 364 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2690 : "not falling back to elementwise accesses\n");
2691 : return false;
2692 : }
2693 :
2694 : /* For BB vectorization build up the vector from existing scalar defs. */
2695 1298021 : if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE)
2696 : return false;
2697 :
2698 : /* Some loads need to explicitly permute the loaded data if there
2699 : is a load permutation. Among those are:
2700 : - VMAT_ELEMENTWISE.
2701 : - VMAT_STRIDED_SLP.
2702 : - VMAT_GATHER_SCATTER:
2703 : - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1).
2704 : - Grouped strided gather (ditto but for #lanes > 1).
2705 :
2706 : For VMAT_ELEMENTWISE we can fold the load permutation into the
2707 : individual indices we access directly, eliding the permutation.
2708 : Strided gather only allows load permutations for the
2709 : single-element case. */
2710 :
2711 1298021 : if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
2712 1298021 : && !(*memory_access_type == VMAT_ELEMENTWISE
2713 53548 : || (mat_gather_scatter_p (*memory_access_type)
2714 0 : && SLP_TREE_LANES (slp_node) == 1
2715 0 : && single_element_p)))
2716 : {
2717 53548 : if (!loop_vinfo)
2718 : {
2719 : /* In BB vectorization we may not actually use a loaded vector
2720 : accessing elements in excess of DR_GROUP_SIZE. */
2721 31554 : stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
2722 31554 : group_info = DR_GROUP_FIRST_ELEMENT (group_info);
2723 31554 : unsigned HOST_WIDE_INT nunits;
2724 31554 : unsigned j, k, maxk = 0;
2725 110360 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k)
2726 78806 : if (k > maxk)
2727 : maxk = k;
2728 31554 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2729 57423 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits)
2730 31554 : || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1)))
2731 : {
2732 5685 : if (dump_enabled_p ())
2733 39 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2734 : "BB vectorization with gaps at the end of "
2735 : "a load is not supported\n");
2736 1404458 : return false;
2737 : }
2738 : }
2739 :
2740 47863 : if (!perm_ok)
2741 : {
2742 2339 : if (dump_enabled_p ())
2743 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
2744 : vect_location,
2745 : "unsupported load permutation\n");
2746 : return false;
2747 : }
2748 :
2749 45524 : *slp_perm = true;
2750 : }
2751 :
2752 : return true;
2753 : }
2754 :
2755 : /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2756 : conditional operation STMT_INFO. When returning true, store the mask
2757 : in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
2758 : the vectorized mask in *MASK_VECTYPE_OUT. */
2759 :
2760 : static bool
2761 12331 : vect_check_scalar_mask (vec_info *vinfo,
2762 : slp_tree slp_node, unsigned mask_index,
2763 : slp_tree *mask_node,
2764 : vect_def_type *mask_dt_out, tree *mask_vectype_out)
2765 : {
2766 12331 : enum vect_def_type mask_dt;
2767 12331 : tree mask_vectype;
2768 12331 : slp_tree mask_node_1;
2769 12331 : tree mask_;
2770 12331 : if (!vect_is_simple_use (vinfo, slp_node, mask_index,
2771 : &mask_, &mask_node_1, &mask_dt, &mask_vectype))
2772 : {
2773 0 : if (dump_enabled_p ())
2774 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2775 : "mask use not simple.\n");
2776 : return false;
2777 : }
2778 :
2779 12331 : if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
2780 12331 : && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
2781 : {
2782 0 : if (dump_enabled_p ())
2783 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2784 : "mask argument is not a boolean.\n");
2785 : return false;
2786 : }
2787 :
2788 12331 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2789 12331 : if (!mask_vectype)
2790 19 : mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
2791 : mask_node_1);
2792 :
2793 12331 : if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2794 : {
2795 0 : if (dump_enabled_p ())
2796 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2797 : "could not find an appropriate vector mask type.\n");
2798 : return false;
2799 : }
2800 :
2801 12331 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2802 24662 : TYPE_VECTOR_SUBPARTS (vectype)))
2803 : {
2804 0 : if (dump_enabled_p ())
2805 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2806 : "vector mask type %T"
2807 : " does not match vector data type %T.\n",
2808 : mask_vectype, vectype);
2809 :
2810 : return false;
2811 : }
2812 :
2813 12331 : *mask_dt_out = mask_dt;
2814 12331 : *mask_vectype_out = mask_vectype;
2815 12331 : *mask_node = mask_node_1;
2816 12331 : return true;
2817 : }
2818 :
2819 :
2820 : /* Return true if stored value is suitable for vectorizing store
2821 : statement STMT_INFO. When returning true, store the scalar stored
2822 : in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2823 : the type of the vectorized store value in
2824 : *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2825 :
2826 : static bool
2827 1388596 : vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
2828 : slp_tree slp_node, slp_tree *rhs_node,
2829 : vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
2830 : vec_load_store_type *vls_type_out)
2831 : {
2832 1388596 : int op_no = 0;
2833 1388596 : if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
2834 : {
2835 1857 : if (gimple_call_internal_p (call)
2836 1857 : && internal_store_fn_p (gimple_call_internal_fn (call)))
2837 1857 : op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
2838 : }
2839 1388596 : op_no = vect_slp_child_index_for_operand (stmt_info, op_no);
2840 :
2841 1388596 : enum vect_def_type rhs_dt;
2842 1388596 : tree rhs_vectype;
2843 1388596 : tree rhs;
2844 1388596 : if (!vect_is_simple_use (vinfo, slp_node, op_no,
2845 : &rhs, rhs_node, &rhs_dt, &rhs_vectype))
2846 : {
2847 0 : if (dump_enabled_p ())
2848 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2849 : "use not simple.\n");
2850 : return false;
2851 : }
2852 :
2853 : /* In the case this is a store from a constant make sure
2854 : native_encode_expr can handle it. */
2855 1388596 : if (rhs_dt == vect_constant_def
2856 1388596 : && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2857 : {
2858 0 : if (dump_enabled_p ())
2859 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2860 : "cannot encode constant as a byte sequence.\n");
2861 : return false;
2862 : }
2863 :
2864 1388596 : tree vectype = SLP_TREE_VECTYPE (slp_node);
2865 1388596 : if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2866 : {
2867 24 : if (dump_enabled_p ())
2868 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2869 : "incompatible vector types.\n");
2870 : return false;
2871 : }
2872 :
2873 1388572 : *rhs_dt_out = rhs_dt;
2874 1388572 : *rhs_vectype_out = rhs_vectype;
2875 1388572 : if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2876 1024002 : *vls_type_out = VLS_STORE_INVARIANT;
2877 : else
2878 364570 : *vls_type_out = VLS_STORE;
2879 : return true;
2880 : }
2881 :
2882 : /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2883 : Note that we support masks with floating-point type, in which case the
2884 : floats are interpreted as a bitmask. */
2885 :
2886 : static tree
2887 166 : vect_build_all_ones_mask (vec_info *vinfo,
2888 : stmt_vec_info stmt_info, tree masktype)
2889 : {
2890 166 : if (TREE_CODE (masktype) == INTEGER_TYPE)
2891 98 : return build_int_cst (masktype, -1);
2892 68 : else if (VECTOR_BOOLEAN_TYPE_P (masktype)
2893 136 : || TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2894 : {
2895 19 : tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2896 19 : mask = build_vector_from_val (masktype, mask);
2897 19 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2898 : }
2899 49 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2900 : {
2901 : REAL_VALUE_TYPE r;
2902 : long tmp[6];
2903 343 : for (int j = 0; j < 6; ++j)
2904 294 : tmp[j] = -1;
2905 49 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2906 49 : tree mask = build_real (TREE_TYPE (masktype), r);
2907 49 : mask = build_vector_from_val (masktype, mask);
2908 49 : return vect_init_vector (vinfo, stmt_info, mask, masktype, NULL);
2909 : }
2910 0 : gcc_unreachable ();
2911 : }
2912 :
2913 : /* Build an all-zero merge value of type VECTYPE while vectorizing
2914 : STMT_INFO as a gather load. */
2915 :
2916 : static tree
2917 154 : vect_build_zero_merge_argument (vec_info *vinfo,
2918 : stmt_vec_info stmt_info, tree vectype)
2919 : {
2920 154 : tree merge;
2921 154 : if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2922 49 : merge = build_int_cst (TREE_TYPE (vectype), 0);
2923 105 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2924 : {
2925 : REAL_VALUE_TYPE r;
2926 : long tmp[6];
2927 735 : for (int j = 0; j < 6; ++j)
2928 630 : tmp[j] = 0;
2929 105 : real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2930 105 : merge = build_real (TREE_TYPE (vectype), r);
2931 : }
2932 : else
2933 0 : gcc_unreachable ();
2934 154 : merge = build_vector_from_val (vectype, merge);
2935 154 : return vect_init_vector (vinfo, stmt_info, merge, vectype, NULL);
2936 : }
2937 :
2938 : /* Return the corresponding else value for an else value constant
2939 : ELSVAL with type TYPE. */
2940 :
2941 : tree
2942 1841 : vect_get_mask_load_else (int elsval, tree type)
2943 : {
2944 1841 : tree els;
2945 1841 : if (elsval == MASK_LOAD_ELSE_UNDEFINED)
2946 : {
2947 0 : tree tmp = create_tmp_var (type);
2948 : /* No need to warn about anything. */
2949 0 : TREE_NO_WARNING (tmp) = 1;
2950 0 : els = get_or_create_ssa_default_def (cfun, tmp);
2951 : }
2952 1841 : else if (elsval == MASK_LOAD_ELSE_M1)
2953 0 : els = build_minus_one_cst (type);
2954 1841 : else if (elsval == MASK_LOAD_ELSE_ZERO)
2955 1841 : els = build_zero_cst (type);
2956 : else
2957 0 : gcc_unreachable ();
2958 :
2959 1841 : return els;
2960 : }
2961 :
2962 : /* Build a gather load call while vectorizing STMT_INFO. Insert new
2963 : instructions before GSI and add them to VEC_STMT. GS_INFO describes
2964 : the gather load operation. If the load is conditional, MASK is the
2965 : vectorized condition, otherwise MASK is null. PTR is the base
2966 : pointer and OFFSET is the vectorized offset. */
2967 :
2968 : static gimple *
2969 345 : vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
2970 : slp_tree slp_node, tree vectype,
2971 : gimple_stmt_iterator *gsi, tree decl,
2972 : tree ptr, tree offset, tree mask)
2973 : {
2974 345 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
2975 345 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
2976 345 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2977 345 : /* ptrtype */ arglist = TREE_CHAIN (arglist);
2978 345 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2979 345 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2980 345 : tree scaletype = TREE_VALUE (arglist);
2981 345 : tree var;
2982 345 : gcc_checking_assert (types_compatible_p (srctype, rettype)
2983 : && (!mask
2984 : || TREE_CODE (masktype) == INTEGER_TYPE
2985 : || types_compatible_p (srctype, masktype)));
2986 :
2987 345 : tree op = offset;
2988 345 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
2989 : {
2990 99 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
2991 : TYPE_VECTOR_SUBPARTS (idxtype)));
2992 99 : var = vect_get_new_ssa_name (idxtype, vect_simple_var);
2993 99 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
2994 99 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
2995 99 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
2996 99 : op = var;
2997 : }
2998 :
2999 345 : tree src_op = NULL_TREE;
3000 345 : tree mask_op = NULL_TREE;
3001 345 : if (mask)
3002 : {
3003 191 : if (!useless_type_conversion_p (masktype, TREE_TYPE (mask)))
3004 : {
3005 191 : tree utype, optype = TREE_TYPE (mask);
3006 191 : if (VECTOR_TYPE_P (masktype)
3007 191 : || TYPE_MODE (masktype) == TYPE_MODE (optype))
3008 : utype = masktype;
3009 : else
3010 6 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3011 191 : var = vect_get_new_ssa_name (utype, vect_scalar_var);
3012 191 : tree mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask);
3013 191 : gassign *new_stmt
3014 191 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3015 191 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3016 191 : mask_arg = var;
3017 191 : if (!useless_type_conversion_p (masktype, utype))
3018 : {
3019 6 : gcc_assert (TYPE_PRECISION (utype)
3020 : <= TYPE_PRECISION (masktype));
3021 6 : var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3022 6 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3023 6 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3024 6 : mask_arg = var;
3025 : }
3026 191 : src_op = build_zero_cst (srctype);
3027 191 : mask_op = mask_arg;
3028 : }
3029 : else
3030 : {
3031 : src_op = mask;
3032 : mask_op = mask;
3033 : }
3034 : }
3035 : else
3036 : {
3037 154 : src_op = vect_build_zero_merge_argument (vinfo, stmt_info, rettype);
3038 154 : mask_op = vect_build_all_ones_mask (vinfo, stmt_info, masktype);
3039 : }
3040 :
3041 345 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3042 345 : gimple *new_stmt = gimple_build_call (decl, 5, src_op, ptr, op,
3043 : mask_op, scale);
3044 :
3045 345 : if (!useless_type_conversion_p (vectype, rettype))
3046 : {
3047 52 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
3048 : TYPE_VECTOR_SUBPARTS (rettype)));
3049 52 : op = vect_get_new_ssa_name (rettype, vect_simple_var);
3050 52 : gimple_call_set_lhs (new_stmt, op);
3051 52 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3052 52 : op = build1 (VIEW_CONVERT_EXPR, vectype, op);
3053 52 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR, op);
3054 : }
3055 :
3056 345 : return new_stmt;
3057 : }
3058 :
3059 : /* Build a scatter store call while vectorizing STMT_INFO. Insert new
3060 : instructions before GSI. GS_INFO describes the scatter store operation.
3061 : PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
3062 : vectorized data to store.
3063 : If the store is conditional, MASK is the vectorized condition, otherwise
3064 : MASK is null. */
3065 :
3066 : static gimple *
3067 161 : vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
3068 : slp_tree slp_node,
3069 : gimple_stmt_iterator *gsi,
3070 : tree decl,
3071 : tree ptr, tree offset, tree oprnd, tree mask)
3072 : {
3073 161 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
3074 161 : tree arglist = TYPE_ARG_TYPES (TREE_TYPE (decl));
3075 161 : /* tree ptrtype = TREE_VALUE (arglist); */ arglist = TREE_CHAIN (arglist);
3076 161 : tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3077 161 : tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3078 161 : tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
3079 161 : tree scaletype = TREE_VALUE (arglist);
3080 161 : gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
3081 : && TREE_CODE (rettype) == VOID_TYPE);
3082 :
3083 161 : tree mask_arg = NULL_TREE;
3084 161 : if (mask)
3085 : {
3086 110 : mask_arg = mask;
3087 110 : tree optype = TREE_TYPE (mask_arg);
3088 110 : tree utype;
3089 110 : if (TYPE_MODE (masktype) == TYPE_MODE (optype))
3090 : utype = masktype;
3091 : else
3092 8 : utype = lang_hooks.types.type_for_mode (TYPE_MODE (optype), 1);
3093 110 : tree var = vect_get_new_ssa_name (utype, vect_scalar_var);
3094 110 : mask_arg = build1 (VIEW_CONVERT_EXPR, utype, mask_arg);
3095 110 : gassign *new_stmt
3096 110 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_arg);
3097 110 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3098 110 : mask_arg = var;
3099 110 : if (!useless_type_conversion_p (masktype, utype))
3100 : {
3101 8 : gcc_assert (TYPE_PRECISION (utype) <= TYPE_PRECISION (masktype));
3102 8 : tree var = vect_get_new_ssa_name (masktype, vect_scalar_var);
3103 8 : new_stmt = gimple_build_assign (var, NOP_EXPR, mask_arg);
3104 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3105 8 : mask_arg = var;
3106 : }
3107 : }
3108 : else
3109 : {
3110 51 : mask_arg = build_int_cst (masktype, -1);
3111 51 : mask_arg = vect_init_vector (vinfo, stmt_info, mask_arg, masktype, NULL);
3112 : }
3113 :
3114 161 : tree src = oprnd;
3115 161 : if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
3116 : {
3117 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
3118 : TYPE_VECTOR_SUBPARTS (srctype)));
3119 0 : tree var = vect_get_new_ssa_name (srctype, vect_simple_var);
3120 0 : src = build1 (VIEW_CONVERT_EXPR, srctype, src);
3121 0 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
3122 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3123 0 : src = var;
3124 : }
3125 :
3126 161 : tree op = offset;
3127 161 : if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
3128 : {
3129 16 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
3130 : TYPE_VECTOR_SUBPARTS (idxtype)));
3131 16 : tree var = vect_get_new_ssa_name (idxtype, vect_simple_var);
3132 16 : op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
3133 16 : gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
3134 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3135 16 : op = var;
3136 : }
3137 :
3138 161 : tree scale = build_int_cst (scaletype, SLP_TREE_GS_SCALE (slp_node));
3139 161 : gcall *new_stmt
3140 161 : = gimple_build_call (decl, 5, ptr, mask_arg, op, src, scale);
3141 161 : return new_stmt;
3142 : }
3143 :
3144 : /* Prepare the base and offset in GS_INFO for vectorization.
3145 : Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
3146 : to the vectorized offset argument for the first copy of STMT_INFO.
3147 : STMT_INFO is the statement described by GS_INFO and LOOP is the
3148 : containing loop. */
3149 :
3150 : static void
3151 1237 : vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
3152 : tree *dataref_ptr, vec<tree> *vec_offset)
3153 : {
3154 1237 : gimple_seq stmts = NULL;
3155 1237 : *dataref_ptr = force_gimple_operand (SLP_TREE_GS_BASE (slp_node),
3156 : &stmts, true, NULL_TREE);
3157 1237 : if (stmts != NULL)
3158 : {
3159 1008 : basic_block new_bb;
3160 1008 : edge pe = loop_preheader_edge (loop);
3161 1008 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3162 1008 : gcc_assert (!new_bb);
3163 : }
3164 1237 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
3165 1237 : }
3166 :
3167 : /* Prepare to implement a grouped or strided load or store using
3168 : the gather load or scatter store operation described by GS_INFO.
3169 : STMT_INFO is the load or store statement.
3170 :
3171 : Set *DR_STEP to the amount that should be added to pointer base address
3172 : to get to the next iteration's base address.
3173 : Set *DR_BUMP to the amount that should be added to the base
3174 : address after each copy of the vectorized statement in a grouped read.
3175 : Set *VEC_OFFSET to an invariant offset vector in which element I has the
3176 : value I * DR_STEP / SCALE. */
3177 :
3178 : static void
3179 0 : vect_get_strided_load_store_ops (stmt_vec_info stmt_info, slp_tree node,
3180 : tree vectype, tree offset_vectype,
3181 : loop_vec_info loop_vinfo,
3182 : gimple_stmt_iterator *gsi,
3183 : tree *dr_step, tree *dr_bump,
3184 : tree *vec_offset)
3185 : {
3186 0 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
3187 :
3188 0 : tree dr_step_temp
3189 0 : = size_binop (MULT_EXPR,
3190 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3191 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
3192 0 : *dr_step = LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)
3193 0 : ? cse_and_gimplify_to_preheader (loop_vinfo, dr_step_temp)
3194 0 : : force_gimple_operand_gsi (gsi, dr_step_temp, false, NULL_TREE,
3195 : true, GSI_SAME_STMT);
3196 0 : tree bump = size_binop (MULT_EXPR,
3197 : fold_convert (sizetype, unshare_expr (DR_STEP (dr))),
3198 : size_int (TYPE_VECTOR_SUBPARTS (vectype)));
3199 0 : *dr_bump = cse_and_gimplify_to_preheader (loop_vinfo, bump);
3200 :
3201 0 : internal_fn ifn
3202 0 : = DR_IS_READ (dr) ? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
3203 0 : if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
3204 : {
3205 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo,
3206 : unshare_expr (DR_STEP (dr)));
3207 0 : return;
3208 : }
3209 :
3210 : /* The offset given in GS_INFO can have pointer type, so use the element
3211 : type of the vector instead. */
3212 0 : tree offset_type = TREE_TYPE (offset_vectype);
3213 :
3214 : /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3215 0 : tree step = size_binop (EXACT_DIV_EXPR, unshare_expr (DR_STEP (dr)),
3216 : ssize_int (SLP_TREE_GS_SCALE (node)));
3217 0 : step = fold_convert (offset_type, step);
3218 :
3219 : /* Create {0, X, X*2, X*3, ...}. */
3220 0 : tree offset = fold_build2 (VEC_SERIES_EXPR, offset_vectype,
3221 : build_zero_cst (offset_type), step);
3222 0 : *vec_offset = cse_and_gimplify_to_preheader (loop_vinfo, offset);
3223 : }
3224 :
3225 : /* Return the amount that should be added to a vector pointer, represented by
3226 : DR_INFO, to increment to the next vectorized iteration. */
3227 :
3228 : static tree
3229 719642 : vect_get_data_ptr_step (vec_info *vinfo, dr_vec_info *dr_info,
3230 : vect_memory_access_type memory_access_type)
3231 : {
3232 719642 : if (memory_access_type == VMAT_INVARIANT)
3233 0 : return size_zero_node;
3234 :
3235 719642 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3236 :
3237 : /* For BB SLP there is no next iteration. */
3238 719642 : if (!loop_vinfo)
3239 584231 : return build_zero_cst (sizetype);
3240 :
3241 135411 : tree step = vect_dr_behavior (loop_vinfo, dr_info)->step;
3242 :
3243 : /* gather/scatter never reach here. */
3244 135411 : gcc_assert (!mat_gather_scatter_p (memory_access_type));
3245 :
3246 135411 : tree iv_increment = LOOP_VINFO_IV_INCREMENT (loop_vinfo);
3247 :
3248 135411 : return fold_build2 (MULT_EXPR, sizetype, iv_increment,
3249 : fold_convert (sizetype, step));
3250 : }
3251 :
3252 : /* Return the amount that should be added to a vector pointer to move
3253 : to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3254 : being vectorized and MEMORY_ACCESS_TYPE describes the type of
3255 : vectorization. */
3256 :
3257 : static tree
3258 720154 : vect_get_data_ptr_bump (vec_info *vinfo,
3259 : dr_vec_info *dr_info, tree aggr_type,
3260 : vect_memory_access_type memory_access_type)
3261 : {
3262 720154 : if (memory_access_type == VMAT_INVARIANT)
3263 0 : return size_zero_node;
3264 :
3265 720154 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
3266 : /* We do not support SLP loads where num_vec != 1 with SELECT_VL so this value
3267 : should never be needed. */
3268 135923 : if (loop_vinfo && LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
3269 : return NULL_TREE;
3270 :
3271 720154 : tree iv_step = TYPE_SIZE_UNIT (aggr_type);
3272 720154 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
3273 720154 : if (tree_int_cst_sgn (step) == -1)
3274 2840 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
3275 : return iv_step;
3276 : }
3277 :
3278 : /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3279 :
3280 : static bool
3281 152 : vectorizable_bswap (vec_info *vinfo,
3282 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3283 : slp_tree slp_node,
3284 : slp_tree *slp_op,
3285 : tree vectype_in, stmt_vector_for_cost *cost_vec)
3286 : {
3287 152 : tree vectype = SLP_TREE_VECTYPE (slp_node);
3288 152 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3289 :
3290 152 : if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
3291 : {
3292 0 : if (dump_enabled_p ())
3293 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3294 : "mismatched vector sizes %T and %T\n",
3295 : vectype_in, vectype);
3296 : return false;
3297 : }
3298 :
3299 152 : tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
3300 152 : if (! char_vectype)
3301 : return false;
3302 :
3303 152 : poly_uint64 num_bytes = TYPE_VECTOR_SUBPARTS (char_vectype);
3304 152 : unsigned word_bytes;
3305 152 : if (!constant_multiple_p (num_bytes, nunits, &word_bytes))
3306 : return false;
3307 :
3308 : /* The encoding uses one stepped pattern for each byte in the word. */
3309 152 : vec_perm_builder elts (num_bytes, word_bytes, 3);
3310 760 : for (unsigned i = 0; i < 3; ++i)
3311 2712 : for (unsigned j = 0; j < word_bytes; ++j)
3312 2256 : elts.quick_push ((i + 1) * word_bytes - j - 1);
3313 :
3314 152 : vec_perm_indices indices (elts, 1, num_bytes);
3315 152 : machine_mode vmode = TYPE_MODE (char_vectype);
3316 152 : if (!can_vec_perm_const_p (vmode, vmode, indices))
3317 : return false;
3318 :
3319 57 : if (cost_vec)
3320 : {
3321 41 : if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
3322 : {
3323 0 : if (dump_enabled_p ())
3324 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3325 : "incompatible vector types for invariants\n");
3326 : return false;
3327 : }
3328 :
3329 41 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3330 41 : DUMP_VECT_SCOPE ("vectorizable_bswap");
3331 41 : record_stmt_cost (cost_vec,
3332 : 1, vector_stmt, slp_node, 0, vect_prologue);
3333 41 : record_stmt_cost (cost_vec,
3334 41 : vect_get_num_copies (vinfo, slp_node),
3335 : vec_perm, slp_node, 0, vect_body);
3336 41 : return true;
3337 : }
3338 :
3339 16 : tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
3340 :
3341 : /* Transform. */
3342 16 : vec<tree> vec_oprnds = vNULL;
3343 16 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
3344 : /* Arguments are ready. create the new vector stmt. */
3345 16 : unsigned i;
3346 16 : tree vop;
3347 48 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
3348 : {
3349 16 : gimple *new_stmt;
3350 16 : tree tem = make_ssa_name (char_vectype);
3351 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3352 : char_vectype, vop));
3353 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3354 16 : tree tem2 = make_ssa_name (char_vectype);
3355 16 : new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
3356 : tem, tem, bswap_vconst);
3357 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3358 16 : tem = make_ssa_name (vectype);
3359 16 : new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
3360 : vectype, tem2));
3361 16 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3362 16 : slp_node->push_vec_def (new_stmt);
3363 : }
3364 :
3365 16 : vec_oprnds.release ();
3366 16 : return true;
3367 152 : }
3368 :
3369 : /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3370 : integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3371 : in a single step. On success, store the binary pack code in
3372 : *CONVERT_CODE. */
3373 :
3374 : static bool
3375 191 : simple_integer_narrowing (tree vectype_out, tree vectype_in,
3376 : code_helper *convert_code)
3377 : {
3378 382 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
3379 382 : || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
3380 : return false;
3381 :
3382 81 : code_helper code;
3383 81 : int multi_step_cvt = 0;
3384 81 : auto_vec <tree, 8> interm_types;
3385 121 : if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
3386 : &code, &multi_step_cvt, &interm_types)
3387 81 : || multi_step_cvt)
3388 : return false;
3389 :
3390 41 : *convert_code = code;
3391 41 : return true;
3392 81 : }
3393 :
3394 : /* Function vectorizable_call.
3395 :
3396 : Check if STMT_INFO performs a function call that can be vectorized.
3397 : If COST_VEC is passed, calculate costs but don't change anything,
3398 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
3399 : it, and insert it at GSI.
3400 : Return true if STMT_INFO is vectorizable in this way. */
3401 :
3402 : static bool
3403 2644115 : vectorizable_call (vec_info *vinfo,
3404 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
3405 : slp_tree slp_node,
3406 : stmt_vector_for_cost *cost_vec)
3407 : {
3408 2644115 : gcall *stmt;
3409 2644115 : tree vec_dest;
3410 2644115 : tree scalar_dest;
3411 2644115 : tree vec_oprnd0 = NULL_TREE;
3412 2644115 : tree vectype_out, vectype_in;
3413 2644115 : poly_uint64 nunits_in;
3414 2644115 : poly_uint64 nunits_out;
3415 2644115 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3416 2644115 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3417 2644115 : tree fndecl, new_temp, rhs_type;
3418 2644115 : enum vect_def_type dt[5]
3419 : = { vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type,
3420 : vect_unknown_def_type, vect_unknown_def_type };
3421 2644115 : tree vectypes[ARRAY_SIZE (dt)] = {};
3422 2644115 : slp_tree slp_op[ARRAY_SIZE (dt)] = {};
3423 2644115 : auto_vec<tree, 8> vargs;
3424 2644115 : enum { NARROW, NONE, WIDEN } modifier;
3425 2644115 : size_t i, nargs;
3426 2644115 : tree clz_ctz_arg1 = NULL_TREE;
3427 :
3428 2644115 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3429 : return false;
3430 :
3431 2644115 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3432 244891 : && cost_vec)
3433 : return false;
3434 :
3435 : /* Is STMT_INFO a vectorizable call? */
3436 2670247 : stmt = dyn_cast <gcall *> (stmt_info->stmt);
3437 26132 : if (!stmt)
3438 : return false;
3439 :
3440 26132 : if (gimple_call_internal_p (stmt)
3441 26132 : && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3442 16872 : || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3443 : /* Handled by vectorizable_load and vectorizable_store. */
3444 : return false;
3445 :
3446 22650 : if (gimple_call_lhs (stmt) == NULL_TREE
3447 22650 : || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3448 : return false;
3449 :
3450 22644 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
3451 :
3452 22644 : vectype_out = SLP_TREE_VECTYPE (slp_node);
3453 :
3454 : /* Process function arguments. */
3455 22644 : rhs_type = NULL_TREE;
3456 22644 : vectype_in = NULL_TREE;
3457 22644 : nargs = gimple_call_num_args (stmt);
3458 :
3459 : /* Bail out if the function has more than four arguments, we do not have
3460 : interesting builtin functions to vectorize with more than two arguments
3461 : except for fma (cond_fma has more). No arguments is also not good. */
3462 22644 : if (nargs == 0 || nargs > 5)
3463 : return false;
3464 :
3465 : /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3466 22564 : combined_fn cfn = gimple_call_combined_fn (stmt);
3467 22564 : if (cfn == CFN_GOMP_SIMD_LANE)
3468 : {
3469 3203 : nargs = 0;
3470 3203 : rhs_type = unsigned_type_node;
3471 : }
3472 : /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3473 : argument just says whether it is well-defined at zero or not and what
3474 : value should be returned for it. */
3475 22564 : if ((cfn == CFN_CLZ || cfn == CFN_CTZ) && nargs == 2)
3476 : {
3477 168 : nargs = 1;
3478 168 : clz_ctz_arg1 = gimple_call_arg (stmt, 1);
3479 : }
3480 :
3481 22564 : int mask_opno = -1;
3482 22564 : if (internal_fn_p (cfn))
3483 : {
3484 : /* We can only handle direct internal masked calls here,
3485 : vectorizable_simd_clone_call is for the rest. */
3486 19119 : if (cfn == CFN_MASK_CALL)
3487 : return false;
3488 18965 : mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
3489 : }
3490 :
3491 70002 : for (i = 0; i < nargs; i++)
3492 : {
3493 49656 : if ((int) i == mask_opno)
3494 : {
3495 7694 : if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
3496 : &slp_op[i], &dt[i], &vectypes[i]))
3497 : return false;
3498 7694 : continue;
3499 : }
3500 :
3501 41962 : if (!vect_is_simple_use (vinfo, slp_node,
3502 : i, &slp_op[i], &dt[i], &vectypes[i]))
3503 : {
3504 0 : if (dump_enabled_p ())
3505 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3506 : "use not simple.\n");
3507 : return false;
3508 : }
3509 :
3510 : /* We can only handle calls with arguments of the same type. */
3511 41962 : tree op = gimple_call_arg (stmt, i);
3512 41962 : if (rhs_type
3513 41962 : && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3514 : {
3515 2064 : if (dump_enabled_p ())
3516 222 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3517 : "argument types differ.\n");
3518 : return false;
3519 : }
3520 39898 : if (!rhs_type)
3521 19207 : rhs_type = TREE_TYPE (op);
3522 :
3523 39898 : if (!vectype_in)
3524 20467 : vectype_in = vectypes[i];
3525 19431 : else if (vectypes[i]
3526 19431 : && !types_compatible_p (vectypes[i], vectype_in))
3527 : {
3528 0 : if (dump_enabled_p ())
3529 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3530 : "argument vector types differ.\n");
3531 : return false;
3532 : }
3533 : }
3534 : /* If all arguments are external or constant defs, infer the vector type
3535 : from the scalar type. */
3536 20346 : if (!vectype_in)
3537 5822 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
3538 20346 : if (!cost_vec)
3539 4307 : gcc_assert (vectype_in);
3540 16039 : if (!vectype_in)
3541 : {
3542 1122 : if (dump_enabled_p ())
3543 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3544 : "no vectype for scalar type %T\n", rhs_type);
3545 :
3546 : return false;
3547 : }
3548 :
3549 38448 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3550 19224 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
3551 : {
3552 12 : if (dump_enabled_p ())
3553 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3554 : "mixed mask and nonmask vector types\n");
3555 : return false;
3556 : }
3557 :
3558 19212 : if (vect_emulated_vector_p (vectype_in)
3559 19212 : || vect_emulated_vector_p (vectype_out))
3560 : {
3561 0 : if (dump_enabled_p ())
3562 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3563 : "use emulated vector type for call\n");
3564 : return false;
3565 : }
3566 :
3567 : /* FORNOW */
3568 19212 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3569 19212 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3570 19212 : if (known_eq (nunits_in * 2, nunits_out))
3571 : modifier = NARROW;
3572 18630 : else if (known_eq (nunits_out, nunits_in))
3573 : modifier = NONE;
3574 50 : else if (known_eq (nunits_out * 2, nunits_in))
3575 : modifier = WIDEN;
3576 : else
3577 : return false;
3578 :
3579 : /* We only handle functions that do not read or clobber memory. */
3580 38424 : if (gimple_vuse (stmt))
3581 : {
3582 1421 : if (dump_enabled_p ())
3583 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3584 : "function reads from or writes to memory.\n");
3585 : return false;
3586 : }
3587 :
3588 : /* For now, we only vectorize functions if a target specific builtin
3589 : is available. TODO -- in some cases, it might be profitable to
3590 : insert the calls for pieces of the vector, in order to be able
3591 : to vectorize other operations in the loop. */
3592 17791 : fndecl = NULL_TREE;
3593 17791 : internal_fn ifn = IFN_LAST;
3594 17791 : tree callee = gimple_call_fndecl (stmt);
3595 :
3596 : /* First try using an internal function. */
3597 17791 : code_helper convert_code = MAX_TREE_CODES;
3598 17791 : if (cfn != CFN_LAST
3599 17791 : && (modifier == NONE
3600 203 : || (modifier == NARROW
3601 191 : && simple_integer_narrowing (vectype_out, vectype_in,
3602 : &convert_code))))
3603 16760 : ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3604 : vectype_in);
3605 :
3606 : /* Check if the operation traps. */
3607 17791 : bool could_trap = gimple_could_trap_p (STMT_VINFO_STMT (stmt_info));
3608 17791 : if (could_trap && cost_vec && loop_vinfo)
3609 : {
3610 : /* If the operation can trap it must be conditional, otherwise fail. */
3611 528 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3612 528 : ? ifn : get_conditional_internal_fn (ifn));
3613 528 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3614 528 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
3615 : {
3616 : /* We assume that BB SLP fills all lanes, so no inactive lanes can
3617 : cause issues. */
3618 114 : if ((cond_fn == IFN_LAST
3619 76 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3620 : OPTIMIZE_FOR_SPEED))
3621 190 : && (cond_len_fn == IFN_LAST
3622 76 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3623 : OPTIMIZE_FOR_SPEED)))
3624 : {
3625 114 : if (dump_enabled_p ())
3626 10 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3627 : "can't use a fully-masked loop because no"
3628 : " conditional operation is available.\n");
3629 114 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3630 : }
3631 : }
3632 : }
3633 :
3634 : /* If that fails, try asking for a target-specific built-in function. */
3635 17791 : if (ifn == IFN_LAST)
3636 : {
3637 9956 : if (cfn != CFN_LAST)
3638 9087 : fndecl = targetm.vectorize.builtin_vectorized_function
3639 9087 : (cfn, vectype_out, vectype_in);
3640 869 : else if (callee && fndecl_built_in_p (callee, BUILT_IN_MD))
3641 24 : fndecl = targetm.vectorize.builtin_md_vectorized_function
3642 24 : (callee, vectype_out, vectype_in);
3643 : }
3644 :
3645 17791 : if (ifn == IFN_LAST && !fndecl)
3646 : {
3647 9576 : if (cfn == CFN_GOMP_SIMD_LANE
3648 3203 : && SLP_TREE_LANES (slp_node) == 1
3649 3203 : && loop_vinfo
3650 3203 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3651 3203 : && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3652 15982 : && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3653 3203 : == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3654 : {
3655 : /* We can handle IFN_GOMP_SIMD_LANE by returning a
3656 : { 0, 1, 2, ... vf - 1 } vector. */
3657 3203 : gcc_assert (nargs == 0);
3658 : }
3659 6373 : else if (modifier == NONE
3660 6373 : && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3661 6016 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3662 5951 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
3663 5898 : || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
3664 152 : return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
3665 152 : slp_op, vectype_in, cost_vec);
3666 : else
3667 : {
3668 6221 : if (dump_enabled_p ())
3669 262 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3670 : "function is not vectorizable.\n");
3671 : return false;
3672 : }
3673 : }
3674 :
3675 11418 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
3676 11418 : internal_fn cond_fn = (internal_fn_mask_index (ifn) != -1
3677 11418 : ? ifn : get_conditional_internal_fn (ifn));
3678 11418 : internal_fn cond_len_fn = get_len_internal_fn (cond_fn);
3679 11418 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3680 9518 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
3681 11418 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
3682 11418 : if (cost_vec) /* transformation not required. */
3683 : {
3684 22508 : for (i = 0; i < nargs; ++i)
3685 15381 : if (!vect_maybe_update_slp_op_vectype (slp_op[i],
3686 15381 : vectypes[i]
3687 : ? vectypes[i] : vectype_in))
3688 : {
3689 0 : if (dump_enabled_p ())
3690 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3691 : "incompatible vector types for invariants\n");
3692 : return false;
3693 : }
3694 7127 : SLP_TREE_TYPE (slp_node) = call_vec_info_type;
3695 7127 : DUMP_VECT_SCOPE ("vectorizable_call");
3696 7127 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
3697 :
3698 7127 : if (loop_vinfo
3699 6167 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
3700 4070 : && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3701 : {
3702 2558 : if (reduc_idx >= 0
3703 1631 : && (cond_fn == IFN_LAST
3704 1631 : || !direct_internal_fn_supported_p (cond_fn, vectype_out,
3705 : OPTIMIZE_FOR_SPEED))
3706 2570 : && (cond_len_fn == IFN_LAST
3707 12 : || !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3708 : OPTIMIZE_FOR_SPEED)))
3709 : {
3710 12 : if (dump_enabled_p ())
3711 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3712 : "can't use a fully-masked loop because no"
3713 : " conditional operation is available.\n");
3714 12 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
3715 : }
3716 : else
3717 : {
3718 2546 : tree scalar_mask = NULL_TREE;
3719 2546 : if (mask_opno >= 0)
3720 2546 : scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3721 2546 : if (cond_len_fn != IFN_LAST
3722 2546 : && direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3723 : OPTIMIZE_FOR_SPEED))
3724 0 : vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3725 : 1);
3726 : else
3727 2546 : vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3728 : scalar_mask);
3729 : }
3730 : }
3731 7127 : return true;
3732 : }
3733 :
3734 : /* Transform. */
3735 :
3736 4291 : if (dump_enabled_p ())
3737 426 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3738 :
3739 : /* Handle def. */
3740 4291 : scalar_dest = gimple_call_lhs (stmt);
3741 4291 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3742 :
3743 4291 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3744 3351 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
3745 4291 : unsigned int vect_nargs = nargs;
3746 4291 : if (len_loop_p && (reduc_idx >= 0 || could_trap || mask_opno >= 0))
3747 : {
3748 0 : ifn = cond_len_fn;
3749 : /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3750 0 : vect_nargs += 2;
3751 : /* But unless there's a mask argument already we need that
3752 : as well, and an else value. */
3753 0 : if (mask_opno == -1)
3754 0 : vect_nargs += 2;
3755 : }
3756 4291 : else if (masked_loop_p && mask_opno == -1 && (reduc_idx >= 0 || could_trap))
3757 : {
3758 0 : ifn = cond_fn;
3759 0 : vect_nargs += 2;
3760 : }
3761 4291 : int len_opno = internal_fn_len_index (ifn);
3762 4291 : if (clz_ctz_arg1)
3763 59 : ++vect_nargs;
3764 :
3765 4291 : if (modifier == NONE || ifn != IFN_LAST)
3766 : {
3767 4259 : tree prev_res = NULL_TREE;
3768 4259 : vargs.safe_grow (vect_nargs, true);
3769 4259 : auto_vec<vec<tree> > vec_defs (nargs);
3770 :
3771 : /* Build argument list for the vectorized call. */
3772 4259 : if (cfn == CFN_GOMP_SIMD_LANE)
3773 : {
3774 3304 : for (i = 0; i < nvectors; ++i)
3775 : {
3776 : /* ??? For multi-lane SLP we'd need to build
3777 : { 0, 0, .., 1, 1, ... }. */
3778 1706 : tree cst = build_index_vector (vectype_out,
3779 : i * nunits_out, 1);
3780 1706 : tree new_var
3781 1706 : = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3782 1706 : gimple *init_stmt = gimple_build_assign (new_var, cst);
3783 1706 : vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
3784 1706 : new_temp = make_ssa_name (vec_dest);
3785 1706 : gimple *new_stmt = gimple_build_assign (new_temp, new_var);
3786 1706 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3787 1706 : slp_node->push_vec_def (new_stmt);
3788 : }
3789 : }
3790 : else
3791 : {
3792 2661 : vec<tree> vec_oprnds0;
3793 2661 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3794 2661 : vec_oprnds0 = vec_defs[0];
3795 :
3796 : /* Arguments are ready. Create the new vector stmt. */
3797 5475 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3798 : {
3799 2814 : int varg = 0;
3800 : /* Add the mask if necessary. */
3801 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3802 2816 : && internal_fn_mask_index (ifn) != -1)
3803 : {
3804 0 : gcc_assert (internal_fn_mask_index (ifn) == varg);
3805 0 : if (masked_loop_p)
3806 : {
3807 0 : unsigned int vec_num = vec_oprnds0.length ();
3808 0 : vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi,
3809 : masks, vec_num,
3810 : vectype_out, i);
3811 : }
3812 : else
3813 : {
3814 0 : tree mask_vectype = truth_type_for (vectype_out);
3815 0 : vargs[varg++] = vect_build_all_ones_mask (loop_vinfo,
3816 : stmt_info,
3817 : mask_vectype);
3818 : }
3819 : }
3820 2814 : size_t k;
3821 10220 : for (k = 0; k < nargs; k++)
3822 : {
3823 7406 : vec<tree> vec_oprndsk = vec_defs[k];
3824 7406 : vargs[varg++] = vec_oprndsk[i];
3825 : }
3826 : /* Add the else value if necessary. */
3827 38 : if ((masked_loop_p || len_loop_p) && mask_opno == -1
3828 2816 : && internal_fn_else_index (ifn) != -1)
3829 : {
3830 0 : gcc_assert (internal_fn_else_index (ifn) == varg);
3831 0 : if (reduc_idx >= 0)
3832 0 : vargs[varg++] = vargs[reduc_idx + 1];
3833 : else
3834 : {
3835 0 : auto else_value = targetm.preferred_else_value
3836 0 : (ifn, vectype_out, varg - 1, &vargs[1]);
3837 0 : vargs[varg++] = else_value;
3838 : }
3839 : }
3840 2814 : if (clz_ctz_arg1)
3841 59 : vargs[varg++] = clz_ctz_arg1;
3842 :
3843 2814 : gimple *new_stmt;
3844 2814 : if (modifier == NARROW)
3845 : {
3846 : /* We don't define any narrowing conditional functions
3847 : at present. */
3848 0 : gcc_assert (mask_opno < 0);
3849 0 : tree half_res = make_ssa_name (vectype_in);
3850 0 : gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3851 0 : gimple_call_set_lhs (call, half_res);
3852 0 : gimple_call_set_nothrow (call, true);
3853 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3854 0 : if ((i & 1) == 0)
3855 : {
3856 0 : prev_res = half_res;
3857 0 : continue;
3858 : }
3859 0 : new_temp = make_ssa_name (vec_dest);
3860 0 : new_stmt = vect_gimple_build (new_temp, convert_code,
3861 : prev_res, half_res);
3862 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
3863 : }
3864 : else
3865 : {
3866 2814 : if (len_opno >= 0 && len_loop_p)
3867 : {
3868 0 : unsigned int vec_num = vec_oprnds0.length ();
3869 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
3870 : vec_num, vectype_out, i, 1, true);
3871 0 : signed char biasval
3872 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3873 0 : tree bias = build_int_cst (intQI_type_node, biasval);
3874 0 : vargs[len_opno] = len;
3875 0 : vargs[len_opno + 1] = bias;
3876 : }
3877 2814 : else if (mask_opno >= 0 && masked_loop_p)
3878 : {
3879 36 : unsigned int vec_num = vec_oprnds0.length ();
3880 36 : tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
3881 : vec_num, vectype_out, i);
3882 36 : vargs[mask_opno]
3883 72 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
3884 36 : vargs[mask_opno], gsi);
3885 : }
3886 :
3887 2814 : gcall *call;
3888 2814 : if (ifn != IFN_LAST)
3889 2733 : call = gimple_build_call_internal_vec (ifn, vargs);
3890 : else
3891 81 : call = gimple_build_call_vec (fndecl, vargs);
3892 2814 : new_temp = make_ssa_name (vec_dest, call);
3893 2814 : gimple_call_set_lhs (call, new_temp);
3894 2814 : gimple_call_set_nothrow (call, true);
3895 2814 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3896 2814 : new_stmt = call;
3897 : }
3898 2814 : slp_node->push_vec_def (new_stmt);
3899 : }
3900 : }
3901 :
3902 11303 : for (i = 0; i < nargs; i++)
3903 : {
3904 7044 : vec<tree> vec_oprndsi = vec_defs[i];
3905 7044 : vec_oprndsi.release ();
3906 : }
3907 4259 : }
3908 32 : else if (modifier == NARROW)
3909 : {
3910 32 : auto_vec<vec<tree> > vec_defs (nargs);
3911 : /* We don't define any narrowing conditional functions at present. */
3912 32 : gcc_assert (mask_opno < 0);
3913 :
3914 : /* Build argument list for the vectorized call. */
3915 32 : vargs.create (nargs * 2);
3916 :
3917 32 : vect_get_slp_defs (vinfo, slp_node, &vec_defs);
3918 32 : vec<tree> vec_oprnds0 = vec_defs[0];
3919 :
3920 : /* Arguments are ready. Create the new vector stmt. */
3921 64 : for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3922 : {
3923 32 : size_t k;
3924 32 : vargs.truncate (0);
3925 96 : for (k = 0; k < nargs; k++)
3926 : {
3927 32 : vec<tree> vec_oprndsk = vec_defs[k];
3928 32 : vargs.quick_push (vec_oprndsk[i]);
3929 32 : vargs.quick_push (vec_oprndsk[i + 1]);
3930 : }
3931 32 : gcall *call;
3932 32 : if (ifn != IFN_LAST)
3933 : call = gimple_build_call_internal_vec (ifn, vargs);
3934 : else
3935 32 : call = gimple_build_call_vec (fndecl, vargs);
3936 32 : new_temp = make_ssa_name (vec_dest, call);
3937 32 : gimple_call_set_lhs (call, new_temp);
3938 32 : gimple_call_set_nothrow (call, true);
3939 32 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
3940 32 : slp_node->push_vec_def (call);
3941 : }
3942 :
3943 64 : for (i = 0; i < nargs; i++)
3944 : {
3945 32 : vec<tree> vec_oprndsi = vec_defs[i];
3946 32 : vec_oprndsi.release ();
3947 : }
3948 32 : }
3949 : else
3950 : /* No current target implements this case. */
3951 : return false;
3952 :
3953 4291 : vargs.release ();
3954 :
3955 4291 : return true;
3956 2644115 : }
3957 :
3958 :
3959 : struct simd_call_arg_info
3960 : {
3961 : tree vectype;
3962 : tree op;
3963 : HOST_WIDE_INT linear_step;
3964 : enum vect_def_type dt;
3965 : unsigned int align;
3966 : bool simd_lane_linear;
3967 : };
3968 :
3969 : /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3970 : is linear within simd lane (but not within whole loop), note it in
3971 : *ARGINFO. */
3972 :
3973 : static void
3974 15 : vect_simd_lane_linear (tree op, class loop *loop,
3975 : struct simd_call_arg_info *arginfo)
3976 : {
3977 15 : gimple *def_stmt = SSA_NAME_DEF_STMT (op);
3978 :
3979 15 : if (!is_gimple_assign (def_stmt)
3980 15 : || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
3981 27 : || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
3982 : return;
3983 :
3984 12 : tree base = gimple_assign_rhs1 (def_stmt);
3985 12 : HOST_WIDE_INT linear_step = 0;
3986 12 : tree v = gimple_assign_rhs2 (def_stmt);
3987 48 : while (TREE_CODE (v) == SSA_NAME)
3988 : {
3989 36 : tree t;
3990 36 : def_stmt = SSA_NAME_DEF_STMT (v);
3991 36 : if (is_gimple_assign (def_stmt))
3992 24 : switch (gimple_assign_rhs_code (def_stmt))
3993 : {
3994 0 : case PLUS_EXPR:
3995 0 : t = gimple_assign_rhs2 (def_stmt);
3996 0 : if (linear_step || TREE_CODE (t) != INTEGER_CST)
3997 : return;
3998 0 : base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
3999 0 : v = gimple_assign_rhs1 (def_stmt);
4000 0 : continue;
4001 12 : case MULT_EXPR:
4002 12 : t = gimple_assign_rhs2 (def_stmt);
4003 12 : if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
4004 : return;
4005 12 : linear_step = tree_to_shwi (t);
4006 12 : v = gimple_assign_rhs1 (def_stmt);
4007 12 : continue;
4008 12 : CASE_CONVERT:
4009 12 : t = gimple_assign_rhs1 (def_stmt);
4010 12 : if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
4011 12 : || (TYPE_PRECISION (TREE_TYPE (v))
4012 12 : < TYPE_PRECISION (TREE_TYPE (t))))
4013 : return;
4014 12 : if (!linear_step)
4015 0 : linear_step = 1;
4016 12 : v = t;
4017 12 : continue;
4018 : default:
4019 : return;
4020 : }
4021 12 : else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
4022 12 : && loop->simduid
4023 12 : && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
4024 24 : && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
4025 : == loop->simduid))
4026 : {
4027 12 : if (!linear_step)
4028 0 : linear_step = 1;
4029 12 : arginfo->linear_step = linear_step;
4030 12 : arginfo->op = base;
4031 12 : arginfo->simd_lane_linear = true;
4032 12 : return;
4033 : }
4034 : }
4035 : }
4036 :
4037 : /* Function vectorizable_simd_clone_call.
4038 :
4039 : Check if STMT_INFO performs a function call that can be vectorized
4040 : by calling a simd clone of the function.
4041 : If COST_VEC is passed, calculate costs but don't change anything,
4042 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
4043 : it, and insert it at GSI.
4044 : Return true if STMT_INFO is vectorizable in this way. */
4045 :
4046 : static bool
4047 2632998 : vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
4048 : gimple_stmt_iterator *gsi,
4049 : slp_tree slp_node,
4050 : stmt_vector_for_cost *cost_vec)
4051 : {
4052 2632998 : tree vec_dest;
4053 2632998 : tree scalar_dest;
4054 2632998 : tree vec_oprnd0 = NULL_TREE;
4055 2632998 : tree vectype;
4056 2632998 : poly_uint64 nunits;
4057 2632998 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
4058 2632998 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
4059 2632998 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
4060 2632998 : tree fndecl, new_temp;
4061 2632998 : int j;
4062 2632998 : auto_vec<simd_call_arg_info> arginfo;
4063 2632998 : vec<tree> vargs = vNULL;
4064 2632998 : size_t i, nargs;
4065 2632998 : tree rtype, ratype;
4066 2632998 : vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
4067 2632998 : int masked_call_offset = 0;
4068 :
4069 : /* Is STMT a vectorizable call? */
4070 2632998 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
4071 16196 : if (!stmt)
4072 : return false;
4073 :
4074 16196 : fndecl = gimple_call_fndecl (stmt);
4075 16196 : if (fndecl == NULL_TREE
4076 16196 : && gimple_call_internal_p (stmt, IFN_MASK_CALL))
4077 : {
4078 220 : fndecl = gimple_call_arg (stmt, 0);
4079 220 : gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
4080 220 : fndecl = TREE_OPERAND (fndecl, 0);
4081 220 : gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
4082 : masked_call_offset = 1;
4083 : }
4084 15976 : if (fndecl == NULL_TREE)
4085 : return false;
4086 :
4087 5995 : struct cgraph_node *node = cgraph_node::get (fndecl);
4088 5995 : if (node == NULL || node->simd_clones == NULL)
4089 : return false;
4090 :
4091 1482 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4092 : return false;
4093 :
4094 1482 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4095 0 : && cost_vec)
4096 : return false;
4097 :
4098 1482 : if (gimple_call_lhs (stmt)
4099 1482 : && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
4100 : return false;
4101 :
4102 1482 : gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
4103 :
4104 1482 : vectype = SLP_TREE_VECTYPE (slp_node);
4105 :
4106 2634480 : if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
4107 : return false;
4108 :
4109 : /* Process function arguments. */
4110 1482 : nargs = gimple_call_num_args (stmt) - masked_call_offset;
4111 :
4112 : /* Bail out if the function has zero arguments. */
4113 1482 : if (nargs == 0)
4114 : return false;
4115 :
4116 1418 : vect_simd_clone_data _data;
4117 1418 : vect_simd_clone_data &data = slp_node->get_data (_data);
4118 1418 : vec<tree>& simd_clone_info = data.simd_clone_info;
4119 1418 : arginfo.reserve (nargs, true);
4120 1418 : auto_vec<slp_tree> slp_op;
4121 1418 : slp_op.safe_grow_cleared (nargs);
4122 :
4123 5527 : for (i = 0; i < nargs; i++)
4124 : {
4125 2691 : simd_call_arg_info thisarginfo;
4126 2691 : affine_iv iv;
4127 2691 : tree op;
4128 :
4129 2691 : thisarginfo.linear_step = 0;
4130 2691 : thisarginfo.align = 0;
4131 2691 : thisarginfo.op = NULL_TREE;
4132 2691 : thisarginfo.simd_lane_linear = false;
4133 :
4134 5382 : int op_no = vect_slp_child_index_for_operand (stmt_info,
4135 2691 : i + masked_call_offset);
4136 5382 : if (!vect_is_simple_use (vinfo, slp_node,
4137 2691 : op_no, &op, &slp_op[i],
4138 : &thisarginfo.dt, &thisarginfo.vectype)
4139 2691 : || thisarginfo.dt == vect_uninitialized_def)
4140 : {
4141 0 : if (dump_enabled_p ())
4142 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4143 : "use not simple.\n");
4144 0 : return false;
4145 : }
4146 :
4147 2691 : if (thisarginfo.dt == vect_constant_def
4148 2691 : || thisarginfo.dt == vect_external_def)
4149 : {
4150 : /* With SLP we determine the vector type of constants/externals
4151 : at analysis time, handling conflicts via
4152 : vect_maybe_update_slp_op_vectype. At transform time
4153 : we have a vector type recorded for SLP. */
4154 680 : gcc_assert (cost_vec
4155 : || thisarginfo.vectype != NULL_TREE);
4156 : if (cost_vec)
4157 549 : thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
4158 549 : TREE_TYPE (op),
4159 : slp_node);
4160 : }
4161 : else
4162 2011 : gcc_assert (thisarginfo.vectype != NULL_TREE);
4163 :
4164 : /* For linear arguments, the analyze phase should have saved
4165 : the base and step. */
4166 2560 : if (!cost_vec
4167 1586 : && i * 3 + 4 <= simd_clone_info.length ()
4168 2770 : && simd_clone_info[i * 3 + 2])
4169 : {
4170 118 : thisarginfo.linear_step = tree_to_shwi (simd_clone_info[i * 3 + 2]);
4171 118 : thisarginfo.op = simd_clone_info[i * 3 + 1];
4172 118 : thisarginfo.simd_lane_linear
4173 118 : = (simd_clone_info[i * 3 + 3] == boolean_true_node);
4174 : /* If loop has been peeled for alignment, we need to adjust it. */
4175 118 : tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
4176 118 : tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
4177 118 : if (n1 != n2 && !thisarginfo.simd_lane_linear)
4178 : {
4179 0 : tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
4180 0 : tree step = simd_clone_info[i * 3 + 2];
4181 0 : tree opt = TREE_TYPE (thisarginfo.op);
4182 0 : bias = fold_convert (TREE_TYPE (step), bias);
4183 0 : bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
4184 0 : thisarginfo.op
4185 0 : = fold_build2 (POINTER_TYPE_P (opt)
4186 : ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
4187 : thisarginfo.op, bias);
4188 : }
4189 : }
4190 2573 : else if (cost_vec
4191 1898 : && thisarginfo.dt != vect_constant_def
4192 1771 : && thisarginfo.dt != vect_external_def
4193 1349 : && loop_vinfo
4194 1344 : && SLP_TREE_LANES (slp_node) == 1
4195 1320 : && TREE_CODE (op) == SSA_NAME
4196 2640 : && simple_iv (loop, loop_containing_stmt (stmt), op,
4197 : &iv, false)
4198 2785 : && tree_fits_shwi_p (iv.step))
4199 : {
4200 212 : thisarginfo.linear_step = tree_to_shwi (iv.step);
4201 212 : thisarginfo.op = iv.base;
4202 : }
4203 2361 : else if ((thisarginfo.dt == vect_constant_def
4204 2361 : || thisarginfo.dt == vect_external_def)
4205 680 : && SLP_TREE_LANES (slp_node) == 1
4206 2667 : && POINTER_TYPE_P (TREE_TYPE (op)))
4207 86 : thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
4208 : /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4209 : linear too. */
4210 2691 : if (SLP_TREE_LANES (slp_node) == 1
4211 2271 : && POINTER_TYPE_P (TREE_TYPE (op))
4212 196 : && !thisarginfo.linear_step
4213 112 : && cost_vec
4214 58 : && thisarginfo.dt != vect_constant_def
4215 58 : && thisarginfo.dt != vect_external_def
4216 15 : && loop_vinfo
4217 2706 : && TREE_CODE (op) == SSA_NAME)
4218 15 : vect_simd_lane_linear (op, loop, &thisarginfo);
4219 :
4220 2691 : if (!vectype)
4221 12 : vectype = thisarginfo.vectype;
4222 2691 : arginfo.quick_push (thisarginfo);
4223 : }
4224 :
4225 1418 : poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
4226 1418 : unsigned group_size = SLP_TREE_LANES (slp_node);
4227 1418 : unsigned int badness = 0;
4228 1418 : unsigned int badness_inbranch = 0;
4229 1418 : struct cgraph_node *bestn = NULL;
4230 1418 : struct cgraph_node *bestn_inbranch = NULL;
4231 1418 : if (!cost_vec)
4232 358 : bestn = ((loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4233 358 : ? data.clone_inbranch : data.clone);
4234 : else
4235 6048 : for (struct cgraph_node *n = node->simd_clones; n != NULL;
4236 4988 : n = n->simdclone->next_clone)
4237 : {
4238 4988 : unsigned int this_badness = 0;
4239 4988 : unsigned int num_calls;
4240 : /* The number of arguments in the call and the number of parameters in
4241 : the simdclone should match. However, when the simdclone is
4242 : 'inbranch', it could have one more parameter than nargs when using
4243 : an inbranch simdclone to call a non-inbranch call, either in a
4244 : non-masked loop using a all true constant mask, or inside a masked
4245 : loop using it's mask. */
4246 4988 : size_t simd_nargs = n->simdclone->nargs;
4247 4988 : if (!masked_call_offset && n->simdclone->inbranch)
4248 2223 : simd_nargs--;
4249 4988 : if (!constant_multiple_p (vf * group_size, n->simdclone->simdlen,
4250 : &num_calls)
4251 1948 : || (!n->simdclone->inbranch && (masked_call_offset > 0))
4252 1764 : || (nargs != simd_nargs))
4253 3224 : continue;
4254 1764 : if (num_calls != 1)
4255 1162 : this_badness += floor_log2 (num_calls) * 4096;
4256 1764 : if (n->simdclone->inbranch)
4257 729 : this_badness += 8192;
4258 :
4259 : /* If SLP_TREE_VECTYPE has not been set yet pass the general vector
4260 : mode, which for targets that use it will determine what ISA we can
4261 : vectorize this code with. */
4262 1764 : machine_mode vector_mode = vinfo->vector_mode;
4263 1764 : if (vectype)
4264 1764 : vector_mode = TYPE_MODE (vectype);
4265 1764 : int target_badness = targetm.simd_clone.usable (n, vector_mode);
4266 1764 : if (target_badness < 0)
4267 380 : continue;
4268 1384 : this_badness += target_badness * 512;
4269 4184 : for (i = 0; i < nargs; i++)
4270 : {
4271 3048 : switch (n->simdclone->args[i].arg_type)
4272 : {
4273 2118 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4274 2118 : if (VECTOR_BOOLEAN_TYPE_P (n->simdclone->args[i].vector_type))
4275 : /* Vector mask arguments are not supported. */
4276 : i = -1;
4277 2110 : else if (!useless_type_conversion_p
4278 2110 : (n->simdclone->args[i].orig_type,
4279 2110 : TREE_TYPE (gimple_call_arg (stmt,
4280 : i + masked_call_offset))))
4281 : i = -1;
4282 2110 : else if (arginfo[i].dt == vect_constant_def
4283 2003 : || arginfo[i].dt == vect_external_def
4284 4049 : || arginfo[i].linear_step)
4285 399 : this_badness += 64;
4286 : break;
4287 310 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4288 310 : if ((arginfo[i].dt != vect_constant_def
4289 145 : && arginfo[i].dt != vect_external_def)
4290 410 : || SLP_TREE_LANES (slp_node) != 1)
4291 : i = -1;
4292 : break;
4293 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4294 324 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4295 324 : if (arginfo[i].dt == vect_constant_def
4296 324 : || arginfo[i].dt == vect_external_def
4297 324 : || (arginfo[i].linear_step
4298 324 : != n->simdclone->args[i].linear_step))
4299 : i = -1;
4300 : break;
4301 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4302 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4303 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4304 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4305 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4306 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4307 : /* FORNOW */
4308 : i = -1;
4309 : break;
4310 296 : case SIMD_CLONE_ARG_TYPE_MASK:
4311 296 : if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4312 264 : && n->simdclone->mask_mode != VOIDmode)
4313 : i = -1;
4314 : /* While we can create a traditional data vector from
4315 : an incoming integer mode mask we have no good way to
4316 : force generate an integer mode mask from a traditional
4317 : boolean vector input. */
4318 296 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4319 296 : && !SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4320 : i = -1;
4321 290 : else if (n->simdclone->mask_mode == VOIDmode
4322 : /* FORNOW we only have partial support for vector-type
4323 : masks that can't hold all of simdlen. */
4324 554 : && (maybe_ne (TYPE_VECTOR_SUBPARTS (n->simdclone->args[i].vector_type),
4325 429 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
4326 : /* Verify we can compute the mask argument. */
4327 111 : || !expand_vec_cond_expr_p (n->simdclone->args[i].vector_type,
4328 111 : arginfo[i].vectype)))
4329 : i = -1;
4330 125 : else if (SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4331 : /* FORNOW we only have partial support for
4332 : integer-type masks that represent the same number
4333 : of lanes as the vectorized mask inputs. */
4334 151 : && maybe_ne (exact_div (n->simdclone->simdlen,
4335 : n->simdclone->args[i].linear_step),
4336 26 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4337 : i = -1;
4338 107 : else if (!SCALAR_INT_MODE_P (n->simdclone->mask_mode)
4339 107 : && SCALAR_INT_MODE_P (TYPE_MODE (arginfo[i].vectype)))
4340 8 : this_badness += 2048;
4341 : break;
4342 : }
4343 18 : if (i == (size_t) -1)
4344 : break;
4345 2800 : if (n->simdclone->args[i].alignment > arginfo[i].align)
4346 : {
4347 : i = -1;
4348 : break;
4349 : }
4350 2800 : if (arginfo[i].align)
4351 110 : this_badness += (exact_log2 (arginfo[i].align)
4352 160 : - exact_log2 (n->simdclone->args[i].alignment));
4353 : }
4354 1384 : if (i == (size_t) -1)
4355 248 : continue;
4356 1136 : if (masked_call_offset == 0
4357 1029 : && n->simdclone->inbranch
4358 311 : && n->simdclone->nargs > nargs)
4359 : {
4360 311 : gcc_assert (n->simdclone->args[n->simdclone->nargs - 1].arg_type ==
4361 : SIMD_CLONE_ARG_TYPE_MASK);
4362 : /* Penalize using a masked SIMD clone in a non-masked loop, that is
4363 : not in a branch, as we'd have to construct an all-true mask. */
4364 311 : this_badness += 64;
4365 : }
4366 1136 : if (bestn == NULL || this_badness < badness)
4367 : {
4368 815 : bestn = n;
4369 815 : badness = this_badness;
4370 : }
4371 1136 : if (n->simdclone->inbranch
4372 418 : && (bestn_inbranch == NULL || this_badness < badness_inbranch))
4373 : {
4374 4988 : bestn_inbranch = n;
4375 4988 : badness_inbranch = this_badness;
4376 : }
4377 : }
4378 :
4379 1418 : if (bestn == NULL)
4380 : return false;
4381 :
4382 835 : fndecl = bestn->decl;
4383 835 : nunits = bestn->simdclone->simdlen;
4384 835 : int ncopies = vector_unroll_factor (vf * group_size, nunits);
4385 :
4386 : /* If the function isn't const, only allow it in simd loops where user
4387 : has asserted that at least nunits consecutive iterations can be
4388 : performed using SIMD instructions. */
4389 830 : if ((loop == NULL || maybe_lt ((unsigned) loop->safelen, nunits))
4390 996 : && gimple_vuse (stmt))
4391 : return false;
4392 :
4393 : /* ncopies is the number of SIMD clone calls we create, since simdlen
4394 : is not necessarily matching nunits of the vector types used, track
4395 : that in ncopies_in. */
4396 835 : int ncopies_in = vect_get_num_vectors (vf * group_size, vectype);
4397 :
4398 : /* Sanity check: make sure that at least one copy of the vectorized stmt
4399 : needs to be generated. */
4400 835 : gcc_assert (ncopies >= 1);
4401 :
4402 835 : if (cost_vec) /* transformation not required. */
4403 : {
4404 1578 : for (unsigned i = 0; i < nargs; ++i)
4405 1101 : if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
4406 : {
4407 0 : if (dump_enabled_p ())
4408 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4409 : "incompatible vector types for invariants\n");
4410 : return false;
4411 : }
4412 :
4413 477 : if (!bestn_inbranch && loop_vinfo)
4414 : {
4415 270 : if (dump_enabled_p ()
4416 270 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
4417 193 : dump_printf_loc (MSG_NOTE, vect_location,
4418 : "can't use a fully-masked loop because no"
4419 : " masked simd clone was available.\n");
4420 270 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
4421 : }
4422 :
4423 : /* When the original call is pure or const but the SIMD ABI dictates
4424 : an aggregate return we will have to use a virtual definition and
4425 : in a loop eventually even need to add a virtual PHI. That's
4426 : not straight-forward so allow to fix this up via renaming. */
4427 477 : if (gimple_call_lhs (stmt)
4428 471 : && !gimple_vdef (stmt)
4429 844 : && TREE_CODE (TREE_TYPE (TREE_TYPE (bestn->decl))) == ARRAY_TYPE)
4430 33 : vinfo->any_known_not_updated_vssa = true;
4431 : /* ??? For SLP code-gen we end up inserting after the last
4432 : vector argument def rather than at the original call position
4433 : so automagic virtual operand updating doesn't work. */
4434 954 : if (gimple_vuse (stmt))
4435 147 : vinfo->any_known_not_updated_vssa = true;
4436 :
4437 477 : data.clone = bestn;
4438 477 : data.clone_inbranch = bestn_inbranch;
4439 :
4440 477 : simd_clone_info.safe_push (NULL_TREE);
4441 1715 : for (i = 0;
4442 2642 : i < (bestn_inbranch ? bestn_inbranch : bestn)->simdclone->nargs; i++)
4443 : {
4444 1238 : if (loop_vinfo
4445 1232 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
4446 458 : && (bestn_inbranch->simdclone->args[i].arg_type
4447 : == SIMD_CLONE_ARG_TYPE_MASK))
4448 : {
4449 162 : if (masked_call_offset)
4450 : /* When there is an explicit mask we require the
4451 : number of elements to match up. */
4452 49 : vect_record_loop_mask (loop_vinfo,
4453 : &LOOP_VINFO_MASKS (loop_vinfo),
4454 : ncopies_in, vectype, NULL_TREE);
4455 : else
4456 : {
4457 : /* When there is no explicit mask on the call we have
4458 : more relaxed requirements. */
4459 113 : tree masktype;
4460 113 : poly_uint64 callee_nelements;
4461 113 : if (SCALAR_INT_MODE_P (bestn_inbranch->simdclone->mask_mode))
4462 : {
4463 12 : callee_nelements
4464 12 : = exact_div (bestn_inbranch->simdclone->simdlen,
4465 : bestn_inbranch->simdclone->args[i].linear_step);
4466 12 : masktype = get_related_vectype_for_scalar_type
4467 12 : (vinfo->vector_mode, TREE_TYPE (vectype),
4468 : callee_nelements);
4469 : }
4470 : else
4471 : {
4472 101 : masktype = bestn_inbranch->simdclone->args[i].vector_type;
4473 : /* The aarch64 port will add custom attributes to types
4474 : for SVE simdclones which make the types different. We
4475 : should use canonincal types for masks within the
4476 : vectorizer, hence we construct the related vectype
4477 : here. */
4478 101 : masktype
4479 : = build_truth_vector_type_for_mode
4480 101 : (TYPE_VECTOR_SUBPARTS (masktype),
4481 101 : TYPE_MODE (masktype));
4482 101 : callee_nelements = TYPE_VECTOR_SUBPARTS (masktype);
4483 : }
4484 113 : auto o = vector_unroll_factor (nunits, callee_nelements);
4485 113 : vect_record_loop_mask (loop_vinfo,
4486 : &LOOP_VINFO_MASKS (loop_vinfo),
4487 : ncopies * o, masktype, NULL_TREE);
4488 : }
4489 : }
4490 1076 : else if ((bestn->simdclone->args[i].arg_type
4491 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4492 969 : || (bestn->simdclone->args[i].arg_type
4493 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
4494 958 : || (bestn_inbranch
4495 352 : && ((bestn_inbranch->simdclone->args[i].arg_type
4496 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
4497 352 : || (bestn_inbranch->simdclone->args[i].arg_type
4498 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))))
4499 : {
4500 118 : simd_clone_info.safe_grow_cleared (i * 3 + 1, true);
4501 118 : simd_clone_info.safe_push (arginfo[i].op);
4502 202 : tree lst = (POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
4503 202 : ? size_type_node : TREE_TYPE (arginfo[i].op));
4504 118 : tree ls = build_int_cst (lst, arginfo[i].linear_step);
4505 118 : simd_clone_info.safe_push (ls);
4506 118 : tree sll = (arginfo[i].simd_lane_linear
4507 118 : ? boolean_true_node : boolean_false_node);
4508 118 : simd_clone_info.safe_push (sll);
4509 : }
4510 : }
4511 :
4512 477 : SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
4513 477 : slp_node->data = new vect_simd_clone_data (std::move (_data));
4514 477 : DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4515 : /* ??? We're confused by calls w/o LHS. */
4516 477 : if (SLP_TREE_VECTYPE (slp_node))
4517 471 : vect_model_simple_cost (vinfo, ncopies, slp_node, cost_vec);
4518 477 : return true;
4519 : }
4520 :
4521 : /* Transform. */
4522 :
4523 358 : if (dump_enabled_p ())
4524 242 : dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
4525 :
4526 : /* Handle def. */
4527 358 : scalar_dest = gimple_call_lhs (stmt);
4528 358 : vec_dest = NULL_TREE;
4529 358 : rtype = NULL_TREE;
4530 358 : ratype = NULL_TREE;
4531 358 : if (scalar_dest)
4532 : {
4533 352 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
4534 352 : rtype = TREE_TYPE (TREE_TYPE (fndecl));
4535 352 : if (TREE_CODE (rtype) == ARRAY_TYPE)
4536 : {
4537 9 : ratype = rtype;
4538 9 : rtype = TREE_TYPE (ratype);
4539 : }
4540 : }
4541 :
4542 716 : auto_vec<vec<tree> > vec_oprnds;
4543 358 : auto_vec<unsigned> vec_oprnds_i;
4544 358 : vec_oprnds_i.safe_grow_cleared (nargs, true);
4545 358 : vec_oprnds.reserve_exact (nargs);
4546 358 : vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
4547 1181 : for (j = 0; j < ncopies; ++j)
4548 : {
4549 465 : poly_uint64 callee_nelements;
4550 465 : poly_uint64 caller_nelements;
4551 : /* Build argument list for the vectorized call. */
4552 465 : if (j == 0)
4553 358 : vargs.create (nargs);
4554 : else
4555 107 : vargs.truncate (0);
4556 :
4557 1568 : for (i = 0; i < nargs; i++)
4558 : {
4559 1103 : unsigned int k, l, m, o;
4560 1103 : tree atype;
4561 1103 : tree op = gimple_call_arg (stmt, i + masked_call_offset);
4562 1103 : switch (bestn->simdclone->args[i].arg_type)
4563 : {
4564 814 : case SIMD_CLONE_ARG_TYPE_VECTOR:
4565 814 : atype = bestn->simdclone->args[i].vector_type;
4566 814 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4567 814 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4568 814 : o = vector_unroll_factor (nunits, callee_nelements);
4569 1858 : for (m = j * o; m < (j + 1) * o; m++)
4570 : {
4571 1044 : if (known_lt (callee_nelements, caller_nelements))
4572 : {
4573 516 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
4574 258 : if (!constant_multiple_p (caller_nelements,
4575 : callee_nelements, &k))
4576 0 : gcc_unreachable ();
4577 :
4578 258 : gcc_assert ((k & (k - 1)) == 0);
4579 258 : if (m == 0)
4580 : {
4581 57 : vec_oprnds_i[i] = 0;
4582 57 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4583 : }
4584 : else
4585 : {
4586 201 : vec_oprnd0 = arginfo[i].op;
4587 201 : if ((m & (k - 1)) == 0)
4588 72 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4589 : }
4590 258 : arginfo[i].op = vec_oprnd0;
4591 258 : vec_oprnd0
4592 258 : = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
4593 258 : bitsize_int (prec),
4594 258 : bitsize_int ((m & (k - 1)) * prec));
4595 258 : gassign *new_stmt
4596 258 : = gimple_build_assign (make_ssa_name (atype),
4597 : vec_oprnd0);
4598 258 : vect_finish_stmt_generation (vinfo, stmt_info,
4599 : new_stmt, gsi);
4600 258 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4601 : }
4602 : else
4603 : {
4604 786 : if (!constant_multiple_p (callee_nelements,
4605 : caller_nelements, &k))
4606 0 : gcc_unreachable ();
4607 786 : gcc_assert ((k & (k - 1)) == 0);
4608 786 : vec<constructor_elt, va_gc> *ctor_elts;
4609 786 : if (k != 1)
4610 12 : vec_alloc (ctor_elts, k);
4611 : else
4612 774 : ctor_elts = NULL;
4613 810 : for (l = 0; l < k; l++)
4614 : {
4615 798 : if (m == 0 && l == 0)
4616 : {
4617 450 : vec_oprnds_i[i] = 0;
4618 450 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4619 : }
4620 : else
4621 348 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4622 798 : arginfo[i].op = vec_oprnd0;
4623 798 : if (k == 1)
4624 : break;
4625 24 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
4626 : vec_oprnd0);
4627 : }
4628 786 : if (k == 1)
4629 774 : if (!useless_type_conversion_p (TREE_TYPE (vec_oprnd0),
4630 : atype))
4631 : {
4632 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, atype,
4633 : vec_oprnd0);
4634 0 : gassign *new_stmt
4635 0 : = gimple_build_assign (make_ssa_name (atype),
4636 : vec_oprnd0);
4637 0 : vect_finish_stmt_generation (vinfo, stmt_info,
4638 : new_stmt, gsi);
4639 0 : vargs.safe_push (gimple_get_lhs (new_stmt));
4640 : }
4641 : else
4642 774 : vargs.safe_push (vec_oprnd0);
4643 : else
4644 : {
4645 12 : vec_oprnd0 = build_constructor (atype, ctor_elts);
4646 12 : gassign *new_stmt
4647 12 : = gimple_build_assign (make_ssa_name (atype),
4648 : vec_oprnd0);
4649 12 : vect_finish_stmt_generation (vinfo, stmt_info,
4650 : new_stmt, gsi);
4651 12 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4652 : }
4653 : }
4654 : }
4655 : break;
4656 66 : case SIMD_CLONE_ARG_TYPE_MASK:
4657 66 : if (bestn->simdclone->mask_mode == VOIDmode)
4658 : {
4659 60 : atype = bestn->simdclone->args[i].vector_type;
4660 60 : tree elt_type = TREE_TYPE (atype);
4661 60 : tree one = fold_convert (elt_type, integer_one_node);
4662 60 : tree zero = fold_convert (elt_type, integer_zero_node);
4663 60 : callee_nelements = TYPE_VECTOR_SUBPARTS (atype);
4664 60 : caller_nelements = TYPE_VECTOR_SUBPARTS (arginfo[i].vectype);
4665 60 : o = vector_unroll_factor (nunits, callee_nelements);
4666 120 : for (m = j * o; m < (j + 1) * o; m++)
4667 : {
4668 60 : if (maybe_lt (callee_nelements, caller_nelements))
4669 : {
4670 : /* The mask type has fewer elements than simdlen. */
4671 :
4672 : /* FORNOW */
4673 0 : gcc_unreachable ();
4674 : }
4675 60 : else if (known_eq (callee_nelements, caller_nelements))
4676 : {
4677 : /* The SIMD clone function has the same number of
4678 : elements as the current function. */
4679 60 : if (m == 0)
4680 60 : vec_oprnds_i[i] = 0;
4681 60 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4682 60 : if (loop_vinfo
4683 60 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4684 : {
4685 0 : vec_loop_masks *loop_masks
4686 : = &LOOP_VINFO_MASKS (loop_vinfo);
4687 0 : tree loop_mask
4688 0 : = vect_get_loop_mask (loop_vinfo, gsi,
4689 : loop_masks, ncopies_in,
4690 0 : vectype, j);
4691 0 : vec_oprnd0
4692 0 : = prepare_vec_mask (loop_vinfo,
4693 0 : TREE_TYPE (loop_mask),
4694 : loop_mask, vec_oprnd0,
4695 : gsi);
4696 0 : loop_vinfo->vec_cond_masked_set.add ({ vec_oprnd0,
4697 : loop_mask });
4698 :
4699 : }
4700 60 : vec_oprnd0
4701 60 : = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
4702 : build_vector_from_val (atype, one),
4703 : build_vector_from_val (atype, zero));
4704 60 : gassign *new_stmt
4705 60 : = gimple_build_assign (make_ssa_name (atype),
4706 : vec_oprnd0);
4707 60 : vect_finish_stmt_generation (vinfo, stmt_info,
4708 : new_stmt, gsi);
4709 60 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4710 : }
4711 : else
4712 : {
4713 : /* The mask type has more elements than simdlen. */
4714 :
4715 : /* FORNOW */
4716 0 : gcc_unreachable ();
4717 : }
4718 : }
4719 : }
4720 6 : else if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4721 : {
4722 6 : atype = bestn->simdclone->args[i].vector_type;
4723 6 : poly_uint64 atype_subparts
4724 6 : = exact_div (bestn->simdclone->simdlen,
4725 : bestn->simdclone->args[i].linear_step);
4726 6 : o = bestn->simdclone->args[i].linear_step;
4727 12 : for (m = j * o; m < (j + 1) * o; m++)
4728 : {
4729 6 : if (m == 0)
4730 6 : vec_oprnds_i[i] = 0;
4731 6 : if (maybe_lt (atype_subparts,
4732 6 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4733 : {
4734 : /* The mask argument has fewer elements than the
4735 : input vector. */
4736 : /* FORNOW */
4737 0 : gcc_unreachable ();
4738 : }
4739 6 : else if (known_eq (atype_subparts,
4740 : TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
4741 : {
4742 6 : vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
4743 6 : if (loop_vinfo
4744 6 : && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4745 : {
4746 1 : vec_loop_masks *loop_masks
4747 : = &LOOP_VINFO_MASKS (loop_vinfo);
4748 1 : tree loop_mask
4749 1 : = vect_get_loop_mask (loop_vinfo, gsi,
4750 : loop_masks, ncopies_in,
4751 : vectype, j);
4752 1 : vec_oprnd0
4753 1 : = prepare_vec_mask (loop_vinfo,
4754 1 : TREE_TYPE (loop_mask),
4755 : loop_mask, vec_oprnd0,
4756 : gsi);
4757 : }
4758 : /* The vector mask argument matches the input
4759 : in the number of lanes, but not necessarily
4760 : in the mode. */
4761 6 : tree st = lang_hooks.types.type_for_mode
4762 6 : (TYPE_MODE (TREE_TYPE (vec_oprnd0)), 1);
4763 6 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, st,
4764 : vec_oprnd0);
4765 6 : gassign *new_stmt
4766 6 : = gimple_build_assign (make_ssa_name (st),
4767 : vec_oprnd0);
4768 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4769 : new_stmt, gsi);
4770 6 : if (!types_compatible_p (atype, st))
4771 : {
4772 6 : new_stmt
4773 6 : = gimple_build_assign (make_ssa_name (atype),
4774 : NOP_EXPR,
4775 : gimple_assign_lhs
4776 : (new_stmt));
4777 6 : vect_finish_stmt_generation (vinfo, stmt_info,
4778 : new_stmt, gsi);
4779 : }
4780 6 : vargs.safe_push (gimple_assign_lhs (new_stmt));
4781 : }
4782 : else
4783 : {
4784 : /* The mask argument has more elements than the
4785 : input vector. */
4786 : /* FORNOW */
4787 0 : gcc_unreachable ();
4788 : }
4789 : }
4790 : }
4791 : else
4792 0 : gcc_unreachable ();
4793 : break;
4794 102 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
4795 102 : vargs.safe_push (op);
4796 102 : break;
4797 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4798 121 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4799 121 : if (j == 0)
4800 : {
4801 118 : gimple_seq stmts;
4802 118 : arginfo[i].op
4803 118 : = force_gimple_operand (unshare_expr (arginfo[i].op),
4804 : &stmts, true, NULL_TREE);
4805 118 : if (stmts != NULL)
4806 : {
4807 0 : basic_block new_bb;
4808 0 : edge pe = loop_preheader_edge (loop);
4809 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4810 0 : gcc_assert (!new_bb);
4811 : }
4812 118 : if (arginfo[i].simd_lane_linear)
4813 : {
4814 6 : vargs.safe_push (arginfo[i].op);
4815 6 : break;
4816 : }
4817 112 : tree phi_res = copy_ssa_name (op);
4818 112 : gphi *new_phi = create_phi_node (phi_res, loop->header);
4819 112 : add_phi_arg (new_phi, arginfo[i].op,
4820 : loop_preheader_edge (loop), UNKNOWN_LOCATION);
4821 112 : enum tree_code code
4822 196 : = POINTER_TYPE_P (TREE_TYPE (op))
4823 112 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4824 196 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4825 196 : ? sizetype : TREE_TYPE (op);
4826 112 : poly_widest_int cst
4827 112 : = wi::mul (bestn->simdclone->args[i].linear_step,
4828 112 : ncopies * nunits);
4829 112 : tree tcst = wide_int_to_tree (type, cst);
4830 112 : tree phi_arg = copy_ssa_name (op);
4831 112 : gassign *new_stmt
4832 112 : = gimple_build_assign (phi_arg, code, phi_res, tcst);
4833 112 : gimple_stmt_iterator si = gsi_after_labels (loop->header);
4834 112 : gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4835 112 : add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4836 : UNKNOWN_LOCATION);
4837 112 : arginfo[i].op = phi_res;
4838 112 : vargs.safe_push (phi_res);
4839 112 : }
4840 : else
4841 : {
4842 3 : enum tree_code code
4843 6 : = POINTER_TYPE_P (TREE_TYPE (op))
4844 3 : ? POINTER_PLUS_EXPR : PLUS_EXPR;
4845 6 : tree type = POINTER_TYPE_P (TREE_TYPE (op))
4846 6 : ? sizetype : TREE_TYPE (op);
4847 3 : poly_widest_int cst
4848 3 : = wi::mul (bestn->simdclone->args[i].linear_step,
4849 3 : j * nunits);
4850 3 : tree tcst = wide_int_to_tree (type, cst);
4851 3 : new_temp = make_ssa_name (TREE_TYPE (op));
4852 3 : gassign *new_stmt
4853 6 : = gimple_build_assign (new_temp, code,
4854 3 : arginfo[i].op, tcst);
4855 3 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4856 3 : vargs.safe_push (new_temp);
4857 3 : }
4858 : break;
4859 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4860 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4861 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4862 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4863 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4864 0 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4865 0 : default:
4866 0 : gcc_unreachable ();
4867 : }
4868 : }
4869 :
4870 465 : if (masked_call_offset == 0
4871 399 : && bestn->simdclone->inbranch
4872 13 : && bestn->simdclone->nargs > nargs)
4873 : {
4874 13 : unsigned long m, o;
4875 13 : size_t mask_i = bestn->simdclone->nargs - 1;
4876 13 : tree mask;
4877 13 : gcc_assert (bestn->simdclone->args[mask_i].arg_type ==
4878 : SIMD_CLONE_ARG_TYPE_MASK);
4879 :
4880 13 : tree mask_argtype = bestn->simdclone->args[mask_i].vector_type;
4881 13 : tree mask_vectype;
4882 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4883 : {
4884 2 : callee_nelements = exact_div (bestn->simdclone->simdlen,
4885 : bestn->simdclone->args[i].linear_step);
4886 2 : mask_vectype = get_related_vectype_for_scalar_type
4887 2 : (vinfo->vector_mode, TREE_TYPE (vectype), callee_nelements);
4888 : }
4889 : else
4890 : {
4891 11 : mask_vectype = mask_argtype;
4892 11 : callee_nelements = TYPE_VECTOR_SUBPARTS (mask_vectype);
4893 : }
4894 13 : o = vector_unroll_factor (nunits, callee_nelements);
4895 26 : for (m = j * o; m < (j + 1) * o; m++)
4896 : {
4897 13 : if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
4898 : {
4899 1 : vec_loop_masks *loop_masks = &LOOP_VINFO_MASKS (loop_vinfo);
4900 1 : mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
4901 : ncopies * o, mask_vectype, m);
4902 : }
4903 : else
4904 12 : mask = vect_build_all_ones_mask (vinfo, stmt_info,
4905 : mask_argtype);
4906 :
4907 13 : gassign *new_stmt;
4908 13 : if (SCALAR_INT_MODE_P (bestn->simdclone->mask_mode))
4909 : {
4910 : /* This means we are dealing with integer mask modes.
4911 : First convert to an integer type with the same size as
4912 : the current vector type. */
4913 2 : unsigned HOST_WIDE_INT intermediate_size
4914 2 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (mask)));
4915 2 : tree mid_int_type =
4916 2 : build_nonstandard_integer_type (intermediate_size, 1);
4917 2 : mask = build1 (VIEW_CONVERT_EXPR, mid_int_type, mask);
4918 2 : new_stmt
4919 2 : = gimple_build_assign (make_ssa_name (mid_int_type),
4920 : mask);
4921 2 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4922 : /* Then zero-extend to the mask mode. */
4923 2 : mask = fold_build1 (NOP_EXPR, mask_argtype,
4924 : gimple_get_lhs (new_stmt));
4925 : }
4926 11 : else if (bestn->simdclone->mask_mode == VOIDmode)
4927 11 : mask = build3 (VEC_COND_EXPR, mask_argtype, mask,
4928 : build_one_cst (mask_argtype),
4929 : build_zero_cst (mask_argtype));
4930 : else
4931 0 : gcc_unreachable ();
4932 :
4933 13 : new_stmt = gimple_build_assign (make_ssa_name (mask_argtype),
4934 : mask);
4935 13 : vect_finish_stmt_generation (vinfo, stmt_info,
4936 : new_stmt, gsi);
4937 13 : mask = gimple_assign_lhs (new_stmt);
4938 13 : vargs.safe_push (mask);
4939 : }
4940 : }
4941 :
4942 465 : gcall *new_call = gimple_build_call_vec (fndecl, vargs);
4943 465 : if (vec_dest)
4944 : {
4945 459 : gcc_assert (ratype
4946 : || known_eq (TYPE_VECTOR_SUBPARTS (rtype), nunits));
4947 459 : if (ratype)
4948 15 : new_temp = create_tmp_var (ratype);
4949 444 : else if (useless_type_conversion_p (vectype, rtype))
4950 424 : new_temp = make_ssa_name (vec_dest, new_call);
4951 : else
4952 20 : new_temp = make_ssa_name (rtype, new_call);
4953 459 : gimple_call_set_lhs (new_call, new_temp);
4954 : }
4955 465 : vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi);
4956 465 : gimple *new_stmt = new_call;
4957 :
4958 465 : if (vec_dest)
4959 : {
4960 459 : if (!multiple_p (TYPE_VECTOR_SUBPARTS (vectype), nunits))
4961 : {
4962 19 : unsigned int k, l;
4963 38 : poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
4964 38 : poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
4965 19 : k = vector_unroll_factor (nunits,
4966 : TYPE_VECTOR_SUBPARTS (vectype));
4967 19 : gcc_assert ((k & (k - 1)) == 0);
4968 69 : for (l = 0; l < k; l++)
4969 : {
4970 50 : tree t;
4971 50 : if (ratype)
4972 : {
4973 42 : t = build_fold_addr_expr (new_temp);
4974 42 : t = build2 (MEM_REF, vectype, t,
4975 42 : build_int_cst (TREE_TYPE (t), l * bytes));
4976 : }
4977 : else
4978 8 : t = build3 (BIT_FIELD_REF, vectype, new_temp,
4979 8 : bitsize_int (prec), bitsize_int (l * prec));
4980 50 : new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
4981 50 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
4982 :
4983 50 : SLP_TREE_VEC_DEFS (slp_node)
4984 50 : .quick_push (gimple_assign_lhs (new_stmt));
4985 : }
4986 :
4987 19 : if (ratype)
4988 15 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
4989 19 : continue;
4990 19 : }
4991 440 : else if (!multiple_p (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
4992 : {
4993 16 : unsigned int k;
4994 16 : if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
4995 16 : TYPE_VECTOR_SUBPARTS (rtype), &k))
4996 0 : gcc_unreachable ();
4997 16 : gcc_assert ((k & (k - 1)) == 0);
4998 16 : if ((j & (k - 1)) == 0)
4999 8 : vec_alloc (ret_ctor_elts, k);
5000 16 : if (ratype)
5001 : {
5002 0 : unsigned int m, o;
5003 0 : o = vector_unroll_factor (nunits,
5004 : TYPE_VECTOR_SUBPARTS (rtype));
5005 0 : for (m = 0; m < o; m++)
5006 : {
5007 0 : tree tem = build4 (ARRAY_REF, rtype, new_temp,
5008 0 : size_int (m), NULL_TREE, NULL_TREE);
5009 0 : new_stmt = gimple_build_assign (make_ssa_name (rtype),
5010 : tem);
5011 0 : vect_finish_stmt_generation (vinfo, stmt_info,
5012 : new_stmt, gsi);
5013 0 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
5014 : gimple_assign_lhs (new_stmt));
5015 : }
5016 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5017 : }
5018 : else
5019 16 : CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
5020 16 : if ((j & (k - 1)) != k - 1)
5021 8 : continue;
5022 8 : vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
5023 8 : new_stmt
5024 8 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5025 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5026 :
5027 8 : SLP_TREE_VEC_DEFS (slp_node)
5028 8 : .quick_push (gimple_assign_lhs (new_stmt));
5029 8 : continue;
5030 8 : }
5031 424 : else if (ratype)
5032 : {
5033 0 : tree t = build_fold_addr_expr (new_temp);
5034 0 : t = build2 (MEM_REF, vectype, t,
5035 0 : build_int_cst (TREE_TYPE (t), 0));
5036 0 : new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t);
5037 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5038 0 : vect_clobber_variable (vinfo, stmt_info, gsi, new_temp);
5039 : }
5040 424 : else if (!useless_type_conversion_p (vectype, rtype))
5041 : {
5042 0 : vec_oprnd0 = build1 (VIEW_CONVERT_EXPR, vectype, new_temp);
5043 0 : new_stmt
5044 0 : = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
5045 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5046 : }
5047 : }
5048 :
5049 430 : if (gimple_get_lhs (new_stmt))
5050 424 : SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
5051 : }
5052 :
5053 1151 : for (i = 0; i < nargs; ++i)
5054 : {
5055 793 : vec<tree> oprndsi = vec_oprnds[i];
5056 793 : oprndsi.release ();
5057 : }
5058 358 : vargs.release ();
5059 :
5060 : /* Mark the clone as no longer being a candidate for GC. */
5061 358 : bestn->gc_candidate = false;
5062 :
5063 358 : return true;
5064 1418 : }
5065 :
5066 :
5067 : /* Function vect_gen_widened_results_half
5068 :
5069 : Create a vector stmt whose code, type, number of arguments, and result
5070 : variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5071 : VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5072 : In the case that CODE is a CALL_EXPR, this means that a call to DECL
5073 : needs to be created (DECL is a function-decl of a target-builtin).
5074 : STMT_INFO is the original scalar stmt that we are vectorizing. */
5075 :
5076 : static gimple *
5077 31915 : vect_gen_widened_results_half (vec_info *vinfo, code_helper ch,
5078 : tree vec_oprnd0, tree vec_oprnd1, int op_type,
5079 : tree vec_dest, gimple_stmt_iterator *gsi,
5080 : stmt_vec_info stmt_info)
5081 : {
5082 31915 : gimple *new_stmt;
5083 31915 : tree new_temp;
5084 :
5085 : /* Generate half of the widened result: */
5086 31915 : if (op_type != binary_op)
5087 30803 : vec_oprnd1 = NULL;
5088 31915 : new_stmt = vect_gimple_build (vec_dest, ch, vec_oprnd0, vec_oprnd1);
5089 31915 : new_temp = make_ssa_name (vec_dest, new_stmt);
5090 31915 : gimple_set_lhs (new_stmt, new_temp);
5091 31915 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5092 :
5093 31915 : return new_stmt;
5094 : }
5095 :
5096 :
5097 : /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5098 : For multi-step conversions store the resulting vectors and call the function
5099 : recursively. When NARROW_SRC_P is true, there's still a conversion after
5100 : narrowing, don't store the vectors in the SLP_NODE or in vector info of
5101 : the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5102 :
5103 : static void
5104 12430 : vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
5105 : int multi_step_cvt,
5106 : stmt_vec_info stmt_info,
5107 : vec<tree> &vec_dsts,
5108 : gimple_stmt_iterator *gsi,
5109 : slp_tree slp_node, code_helper code,
5110 : bool narrow_src_p)
5111 : {
5112 12430 : unsigned int i;
5113 12430 : tree vop0, vop1, new_tmp, vec_dest;
5114 :
5115 12430 : vec_dest = vec_dsts.pop ();
5116 :
5117 29403 : for (i = 0; i < vec_oprnds->length (); i += 2)
5118 : {
5119 : /* Create demotion operation. */
5120 16973 : vop0 = (*vec_oprnds)[i];
5121 16973 : vop1 = (*vec_oprnds)[i + 1];
5122 16973 : gimple *new_stmt = vect_gimple_build (vec_dest, code, vop0, vop1);
5123 16973 : new_tmp = make_ssa_name (vec_dest, new_stmt);
5124 16973 : gimple_set_lhs (new_stmt, new_tmp);
5125 16973 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5126 16973 : if (multi_step_cvt || narrow_src_p)
5127 : /* Store the resulting vector for next recursive call,
5128 : or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5129 6816 : (*vec_oprnds)[i/2] = new_tmp;
5130 : else
5131 : {
5132 : /* This is the last step of the conversion sequence. Store the
5133 : vectors in SLP_NODE. */
5134 10157 : slp_node->push_vec_def (new_stmt);
5135 : }
5136 : }
5137 :
5138 : /* For multi-step demotion operations we first generate demotion operations
5139 : from the source type to the intermediate types, and then combine the
5140 : results (stored in VEC_OPRNDS) in demotion operation to the destination
5141 : type. */
5142 12430 : if (multi_step_cvt)
5143 : {
5144 : /* At each level of recursion we have half of the operands we had at the
5145 : previous level. */
5146 3024 : vec_oprnds->truncate ((i+1)/2);
5147 3024 : vect_create_vectorized_demotion_stmts (vinfo, vec_oprnds,
5148 : multi_step_cvt - 1,
5149 : stmt_info, vec_dsts, gsi,
5150 3024 : slp_node, VEC_PACK_TRUNC_EXPR,
5151 : narrow_src_p);
5152 : }
5153 :
5154 12430 : vec_dsts.quick_push (vec_dest);
5155 12430 : }
5156 :
5157 :
5158 : /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5159 : and VEC_OPRNDS1, for a binary operation associated with scalar statement
5160 : STMT_INFO. For multi-step conversions store the resulting vectors and
5161 : call the function recursively. When NUM_VECTORS is not -1U then only
5162 : NUM_VECTORS will be produced. */
5163 :
5164 : static void
5165 11601 : vect_create_vectorized_promotion_stmts (vec_info *vinfo,
5166 : vec<tree> *vec_oprnds0,
5167 : vec<tree> *vec_oprnds1,
5168 : stmt_vec_info stmt_info, tree vec_dest,
5169 : gimple_stmt_iterator *gsi,
5170 : code_helper ch1,
5171 : code_helper ch2, int op_type,
5172 : unsigned num_vectors)
5173 : {
5174 11601 : int i;
5175 11601 : tree vop0, vop1, new_tmp1, new_tmp2;
5176 11601 : gimple *new_stmt1, *new_stmt2;
5177 11601 : vec<tree> vec_tmp = vNULL;
5178 :
5179 11601 : vec_tmp.create (num_vectors != -1U
5180 1824 : ? num_vectors : 2 * vec_oprnds0->length ());
5181 27564 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5182 : {
5183 15963 : if (op_type == binary_op)
5184 556 : vop1 = (*vec_oprnds1)[i];
5185 : else
5186 : vop1 = NULL_TREE;
5187 :
5188 : /* Generate the two halves of promotion operation. */
5189 15963 : new_stmt1 = vect_gen_widened_results_half (vinfo, ch1, vop0, vop1,
5190 : op_type, vec_dest, gsi,
5191 : stmt_info);
5192 15963 : new_tmp1 = gimple_get_lhs (new_stmt1);
5193 15963 : vec_tmp.quick_push (new_tmp1);
5194 :
5195 15963 : if (vec_tmp.space (1))
5196 : {
5197 15952 : new_stmt2 = vect_gen_widened_results_half (vinfo, ch2, vop0, vop1,
5198 : op_type, vec_dest, gsi,
5199 : stmt_info);
5200 15952 : new_tmp2 = gimple_get_lhs (new_stmt2);
5201 15952 : vec_tmp.quick_push (new_tmp2);
5202 : }
5203 :
5204 15963 : if (!vec_tmp.space (1))
5205 : break;
5206 : }
5207 :
5208 11601 : vec_oprnds0->release ();
5209 11601 : *vec_oprnds0 = vec_tmp;
5210 11601 : }
5211 :
5212 : /* Create vectorized promotion stmts for widening stmts using only half the
5213 : potential vector size for input. */
5214 : static void
5215 44 : vect_create_half_widening_stmts (vec_info *vinfo,
5216 : vec<tree> *vec_oprnds0,
5217 : vec<tree> *vec_oprnds1,
5218 : stmt_vec_info stmt_info, tree vec_dest,
5219 : gimple_stmt_iterator *gsi,
5220 : code_helper code1,
5221 : int op_type)
5222 : {
5223 44 : int i;
5224 44 : tree vop0, vop1;
5225 44 : gimple *new_stmt1;
5226 44 : gimple *new_stmt2;
5227 44 : gimple *new_stmt3;
5228 44 : vec<tree> vec_tmp = vNULL;
5229 :
5230 44 : vec_tmp.create (vec_oprnds0->length ());
5231 132 : FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
5232 : {
5233 44 : tree new_tmp1, new_tmp2, new_tmp3, out_type;
5234 :
5235 44 : gcc_assert (op_type == binary_op);
5236 44 : vop1 = (*vec_oprnds1)[i];
5237 :
5238 : /* Widen the first vector input. */
5239 44 : out_type = TREE_TYPE (vec_dest);
5240 44 : new_tmp1 = make_ssa_name (out_type);
5241 44 : new_stmt1 = gimple_build_assign (new_tmp1, NOP_EXPR, vop0);
5242 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt1, gsi);
5243 44 : if (VECTOR_TYPE_P (TREE_TYPE (vop1)))
5244 : {
5245 : /* Widen the second vector input. */
5246 44 : new_tmp2 = make_ssa_name (out_type);
5247 44 : new_stmt2 = gimple_build_assign (new_tmp2, NOP_EXPR, vop1);
5248 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt2, gsi);
5249 : /* Perform the operation. With both vector inputs widened. */
5250 44 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, new_tmp2);
5251 : }
5252 : else
5253 : {
5254 : /* Perform the operation. With the single vector input widened. */
5255 0 : new_stmt3 = vect_gimple_build (vec_dest, code1, new_tmp1, vop1);
5256 : }
5257 :
5258 44 : new_tmp3 = make_ssa_name (vec_dest, new_stmt3);
5259 44 : gimple_assign_set_lhs (new_stmt3, new_tmp3);
5260 44 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt3, gsi);
5261 :
5262 : /* Store the results for the next step. */
5263 44 : vec_tmp.quick_push (new_tmp3);
5264 : }
5265 :
5266 44 : vec_oprnds0->release ();
5267 44 : *vec_oprnds0 = vec_tmp;
5268 44 : }
5269 :
5270 :
5271 : /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5272 : If COST_VEC is passed, calculate costs but don't change anything,
5273 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5274 : it, and insert it at GSI.
5275 : Return true if STMT_INFO is vectorizable in this way. */
5276 :
5277 : static bool
5278 2656146 : vectorizable_conversion (vec_info *vinfo,
5279 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5280 : slp_tree slp_node,
5281 : stmt_vector_for_cost *cost_vec)
5282 : {
5283 2656146 : tree vec_dest, cvt_op = NULL_TREE;
5284 2656146 : tree scalar_dest;
5285 2656146 : tree_code tc1;
5286 2656146 : code_helper code, code1, code2;
5287 2656146 : code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
5288 2656146 : tree new_temp;
5289 2656146 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5290 2656146 : poly_uint64 nunits_in;
5291 2656146 : poly_uint64 nunits_out;
5292 2656146 : tree vectype_out, vectype_in;
5293 2656146 : int i;
5294 2656146 : tree lhs_type, rhs_type;
5295 : /* For conversions between floating point and integer, there're 2 NARROW
5296 : cases. NARROW_SRC is for FLOAT_EXPR, means
5297 : integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5298 : This is safe when the range of the source integer can fit into the lower
5299 : precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5300 : floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5301 : For other conversions, when there's narrowing, NARROW_DST is used as
5302 : default. */
5303 2656146 : enum { NARROW_SRC, NARROW_DST, NONE, WIDEN } modifier;
5304 2656146 : vec<tree> vec_oprnds0 = vNULL;
5305 2656146 : vec<tree> vec_oprnds1 = vNULL;
5306 2656146 : tree vop0;
5307 2656146 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5308 2656146 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5309 2656146 : int multi_step_cvt = 0;
5310 2656146 : vec<tree> interm_types = vNULL;
5311 2656146 : tree intermediate_type, cvt_type = NULL_TREE;
5312 2656146 : int op_type;
5313 2656146 : unsigned short fltsz;
5314 :
5315 : /* Is STMT a vectorizable conversion? */
5316 :
5317 2656146 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5318 : return false;
5319 :
5320 2656146 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5321 244891 : && cost_vec)
5322 : return false;
5323 :
5324 2411255 : gimple* stmt = stmt_info->stmt;
5325 2411255 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
5326 : return false;
5327 :
5328 2349724 : if (gimple_get_lhs (stmt) == NULL_TREE
5329 2349724 : || TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5330 : return false;
5331 :
5332 1518521 : if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME)
5333 : return false;
5334 :
5335 1518521 : if (is_gimple_assign (stmt))
5336 : {
5337 1505671 : code = gimple_assign_rhs_code (stmt);
5338 1505671 : op_type = TREE_CODE_LENGTH ((tree_code) code);
5339 : }
5340 12850 : else if (gimple_call_internal_p (stmt))
5341 : {
5342 7665 : code = gimple_call_internal_fn (stmt);
5343 7665 : op_type = gimple_call_num_args (stmt);
5344 : }
5345 : else
5346 : return false;
5347 :
5348 1513336 : bool widen_arith = (code == WIDEN_MULT_EXPR
5349 1510759 : || code == WIDEN_LSHIFT_EXPR
5350 3024095 : || widening_fn_p (code));
5351 :
5352 1510759 : if (!widen_arith
5353 1510759 : && !CONVERT_EXPR_CODE_P (code)
5354 : && code != FIX_TRUNC_EXPR
5355 : && code != FLOAT_EXPR)
5356 : return false;
5357 :
5358 : /* Check types of lhs and rhs. */
5359 198586 : scalar_dest = gimple_get_lhs (stmt);
5360 198586 : lhs_type = TREE_TYPE (scalar_dest);
5361 198586 : vectype_out = SLP_TREE_VECTYPE (slp_node);
5362 :
5363 : /* Check the operands of the operation. */
5364 198586 : slp_tree slp_op0, slp_op1 = NULL;
5365 198586 : if (!vect_is_simple_use (vinfo, slp_node,
5366 : 0, &slp_op0, &dt[0], &vectype_in))
5367 : {
5368 0 : if (dump_enabled_p ())
5369 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5370 : "use not simple.\n");
5371 : return false;
5372 : }
5373 :
5374 198586 : rhs_type = TREE_TYPE (gimple_arg (stmt, 0));
5375 198586 : if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
5376 377318 : && !((INTEGRAL_TYPE_P (lhs_type)
5377 164010 : && INTEGRAL_TYPE_P (rhs_type))
5378 : || (SCALAR_FLOAT_TYPE_P (lhs_type)
5379 9878 : && SCALAR_FLOAT_TYPE_P (rhs_type))))
5380 : return false;
5381 :
5382 193742 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5383 173494 : && INTEGRAL_TYPE_P (lhs_type)
5384 335517 : && !type_has_mode_precision_p (lhs_type))
5385 : {
5386 554 : if (dump_enabled_p ())
5387 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5388 : "type conversion to bit-precision unsupported\n");
5389 : return false;
5390 : }
5391 :
5392 : /* _BitInt values are not sign-/zero-extended to mode precision. */
5393 193188 : if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5394 172940 : && TREE_CODE (rhs_type) == BITINT_TYPE
5395 193222 : && !type_has_mode_precision_p (rhs_type))
5396 : {
5397 32 : if (dump_enabled_p ())
5398 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5399 : "type conversion from _BitInt unsupported\n");
5400 : return false;
5401 : }
5402 :
5403 193156 : if (op_type == binary_op)
5404 : {
5405 2577 : gcc_assert (code == WIDEN_MULT_EXPR
5406 : || code == WIDEN_LSHIFT_EXPR
5407 : || widening_fn_p (code));
5408 :
5409 2577 : tree vectype1_in;
5410 2577 : if (!vect_is_simple_use (vinfo, slp_node, 1,
5411 : &slp_op1, &dt[1], &vectype1_in))
5412 : {
5413 0 : if (dump_enabled_p ())
5414 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5415 : "use not simple.\n");
5416 0 : return false;
5417 : }
5418 : /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5419 : OP1. */
5420 2577 : if (!vectype_in)
5421 162 : vectype_in = vectype1_in;
5422 : }
5423 :
5424 : /* If slp_op0 is an external or constant def, infer the vector type
5425 : from the scalar type. */
5426 193156 : if (!cost_vec)
5427 23983 : gcc_assert (vectype_in);
5428 193156 : if (!vectype_in)
5429 24775 : vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
5430 193156 : if (!vectype_in)
5431 : {
5432 279 : if (dump_enabled_p ())
5433 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5434 : "no vectype for scalar type %T\n", rhs_type);
5435 :
5436 : return false;
5437 : }
5438 :
5439 385754 : if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
5440 192877 : != VECTOR_BOOLEAN_TYPE_P (vectype_in))
5441 : {
5442 309 : if (dump_enabled_p ())
5443 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5444 : "can't convert between boolean and non "
5445 : "boolean vectors %T\n", rhs_type);
5446 :
5447 : return false;
5448 : }
5449 :
5450 192568 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
5451 192568 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5452 192568 : if (known_eq (nunits_out, nunits_in))
5453 92625 : if (widen_arith)
5454 : modifier = WIDEN;
5455 : else
5456 192568 : modifier = NONE;
5457 99943 : else if (multiple_p (nunits_out, nunits_in))
5458 : modifier = NARROW_DST;
5459 : else
5460 : {
5461 56457 : gcc_checking_assert (multiple_p (nunits_in, nunits_out));
5462 : modifier = WIDEN;
5463 : }
5464 :
5465 192568 : bool found_mode = false;
5466 192568 : scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
5467 192568 : scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
5468 192568 : opt_scalar_mode rhs_mode_iter;
5469 192568 : auto_vec<std::pair<tree, tree_code>, 2> converts;
5470 192568 : bool evenodd_ok = false;
5471 :
5472 : /* Supportable by target? */
5473 192568 : switch (modifier)
5474 : {
5475 92258 : case NONE:
5476 92258 : if (code != FIX_TRUNC_EXPR
5477 : && code != FLOAT_EXPR
5478 : && !CONVERT_EXPR_CODE_P (code))
5479 : return false;
5480 92258 : gcc_assert (code.is_tree_code ());
5481 92258 : if (supportable_indirect_convert_operation (code,
5482 : vectype_out, vectype_in,
5483 : converts, slp_op0))
5484 : {
5485 85363 : gcc_assert (converts.length () <= 2);
5486 85363 : if (converts.length () == 1)
5487 : {
5488 85289 : code1 = converts[0].second;
5489 85289 : if (CONVERT_EXPR_CODE_P (code)
5490 160096 : && tree_nop_conversion_p (TREE_TYPE (vectype_out),
5491 74807 : TREE_TYPE (vectype_in)))
5492 : /* NOP conversions are handled by vectorizable_assignment. */
5493 : return false;
5494 : }
5495 : else
5496 : {
5497 74 : cvt_type = NULL_TREE;
5498 74 : multi_step_cvt = converts.length () - 1;
5499 74 : codecvt1 = converts[0].second;
5500 74 : code1 = converts[1].second;
5501 74 : interm_types.safe_push (converts[0].first);
5502 : }
5503 : break;
5504 : }
5505 :
5506 : /* FALLTHRU */
5507 6895 : unsupported:
5508 19295 : if (dump_enabled_p ())
5509 564 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5510 : "conversion not supported by target.\n");
5511 : return false;
5512 :
5513 56824 : case WIDEN:
5514 56824 : if (known_eq (nunits_in, nunits_out))
5515 : {
5516 734 : if (!(code.is_tree_code ()
5517 367 : && supportable_half_widening_operation ((tree_code) code,
5518 : vectype_out, vectype_in,
5519 : &tc1)))
5520 123 : goto unsupported;
5521 244 : code1 = tc1;
5522 244 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5523 : break;
5524 : }
5525 : /* Elements in a vector can only be reordered if used in a reduction
5526 : operation only. */
5527 56457 : if (code == WIDEN_MULT_EXPR
5528 2210 : && loop_vinfo
5529 2069 : && !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info)
5530 : /* For a SLP reduction we cannot swizzle lanes, detecting a
5531 : reduction chain isn't possible here. */
5532 58504 : && SLP_TREE_LANES (slp_node) == 1)
5533 : {
5534 : /* ??? There is no way to look for SLP uses, so work on
5535 : the stmt and what the stmt-based cycle detection gives us. */
5536 3890 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
5537 1945 : stmt_vec_info use_stmt_info
5538 1945 : = lhs ? loop_vinfo->lookup_single_use (lhs) : NULL;
5539 1945 : if (use_stmt_info
5540 1800 : && STMT_VINFO_REDUC_DEF (use_stmt_info))
5541 56457 : evenodd_ok = true;
5542 : }
5543 56457 : if (supportable_widening_operation (code, vectype_out, vectype_in,
5544 : evenodd_ok, &code1,
5545 : &code2, &multi_step_cvt,
5546 : &interm_types))
5547 : {
5548 : /* Binary widening operation can only be supported directly by the
5549 : architecture. */
5550 50015 : gcc_assert (!(multi_step_cvt && op_type == binary_op));
5551 : break;
5552 : }
5553 :
5554 6442 : if (code != FLOAT_EXPR
5555 6808 : || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
5556 6259 : goto unsupported;
5557 :
5558 183 : fltsz = GET_MODE_SIZE (lhs_mode);
5559 270 : FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
5560 : {
5561 270 : rhs_mode = rhs_mode_iter.require ();
5562 540 : if (GET_MODE_SIZE (rhs_mode) > fltsz)
5563 : break;
5564 :
5565 270 : cvt_type
5566 270 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5567 270 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5568 270 : if (cvt_type == NULL_TREE)
5569 0 : goto unsupported;
5570 :
5571 540 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5572 : {
5573 78 : gcc_assert (code.is_tree_code ());
5574 78 : if (!supportable_convert_operation ((tree_code) code, vectype_out,
5575 : cvt_type))
5576 22 : goto unsupported;
5577 56 : codecvt1 = code;
5578 : }
5579 192 : else if (!supportable_widening_operation (code, vectype_out,
5580 : cvt_type, evenodd_ok,
5581 : &codecvt1,
5582 : &codecvt2, &multi_step_cvt,
5583 : &interm_types))
5584 87 : continue;
5585 : else
5586 105 : gcc_assert (multi_step_cvt == 0);
5587 :
5588 161 : if (supportable_widening_operation (NOP_EXPR, cvt_type,
5589 : vectype_in, evenodd_ok, &code1,
5590 : &code2, &multi_step_cvt,
5591 : &interm_types))
5592 : {
5593 : found_mode = true;
5594 : break;
5595 : }
5596 : }
5597 :
5598 161 : if (!found_mode)
5599 0 : goto unsupported;
5600 :
5601 322 : if (GET_MODE_SIZE (rhs_mode) == fltsz)
5602 56 : codecvt2 = ERROR_MARK;
5603 : else
5604 : {
5605 105 : multi_step_cvt++;
5606 105 : interm_types.safe_push (cvt_type);
5607 105 : cvt_type = NULL_TREE;
5608 : }
5609 : break;
5610 :
5611 43486 : case NARROW_DST:
5612 43486 : gcc_assert (op_type == unary_op);
5613 43486 : if (supportable_narrowing_operation (code, vectype_out, vectype_in,
5614 : &code1, &multi_step_cvt,
5615 : &interm_types))
5616 : break;
5617 :
5618 18390 : if (GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
5619 986 : goto unsupported;
5620 :
5621 5144 : if (code == FIX_TRUNC_EXPR)
5622 : {
5623 107 : cvt_type
5624 107 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
5625 107 : cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
5626 107 : if (cvt_type == NULL_TREE)
5627 0 : goto unsupported;
5628 107 : if (supportable_convert_operation ((tree_code) code, cvt_type,
5629 : vectype_in))
5630 105 : codecvt1 = code;
5631 : else
5632 2 : goto unsupported;
5633 105 : if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
5634 : &code1, &multi_step_cvt,
5635 : &interm_types))
5636 : break;
5637 : }
5638 : /* If slp_op0 can be represented with low precision integer,
5639 : truncate it to cvt_type and the do FLOAT_EXPR. */
5640 5037 : else if (code == FLOAT_EXPR)
5641 : {
5642 792 : if (cost_vec)
5643 : {
5644 784 : wide_int op_min_value, op_max_value;
5645 784 : tree def;
5646 :
5647 : /* ??? Merge ranges in case of more than one lane. */
5648 784 : if (SLP_TREE_LANES (slp_op0) != 1
5649 133 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
5650 917 : || !vect_get_range_info (def, &op_min_value, &op_max_value))
5651 755 : goto unsupported;
5652 :
5653 29 : if ((wi::min_precision (op_max_value, SIGNED)
5654 29 : > GET_MODE_BITSIZE (lhs_mode))
5655 29 : || (wi::min_precision (op_min_value, SIGNED)
5656 27 : > GET_MODE_BITSIZE (lhs_mode)))
5657 2 : goto unsupported;
5658 784 : }
5659 :
5660 35 : cvt_type
5661 35 : = build_nonstandard_integer_type (GET_MODE_BITSIZE (lhs_mode), 0);
5662 35 : cvt_type = get_same_sized_vectype (cvt_type, vectype_out);
5663 35 : if (cvt_type == NULL_TREE)
5664 0 : goto unsupported;
5665 35 : if (!supportable_narrowing_operation (NOP_EXPR, cvt_type, vectype_in,
5666 : &code1, &multi_step_cvt,
5667 : &interm_types))
5668 2 : goto unsupported;
5669 33 : if (supportable_convert_operation ((tree_code) code, vectype_out,
5670 : cvt_type))
5671 : {
5672 33 : codecvt1 = code;
5673 33 : modifier = NARROW_SRC;
5674 33 : break;
5675 : }
5676 : }
5677 :
5678 4249 : goto unsupported;
5679 :
5680 : default:
5681 : gcc_unreachable ();
5682 : }
5683 :
5684 109610 : if (modifier == WIDEN
5685 109610 : && loop_vinfo
5686 48849 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
5687 130626 : && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
5688 20994 : || widening_evenodd_fn_p (code1)))
5689 : {
5690 22 : if (dump_enabled_p ())
5691 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5692 : "can't use a fully-masked loop because"
5693 : " widening operation on even/odd elements"
5694 : " mixes up lanes.\n");
5695 22 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
5696 : }
5697 :
5698 109610 : if (cost_vec) /* transformation not required. */
5699 : {
5700 85627 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
5701 85627 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
5702 : {
5703 0 : if (dump_enabled_p ())
5704 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5705 : "incompatible vector types for invariants\n");
5706 : return false;
5707 : }
5708 85627 : DUMP_VECT_SCOPE ("vectorizable_conversion");
5709 85627 : unsigned int nvectors = vect_get_num_copies (vinfo, slp_node);
5710 85627 : if (modifier == NONE)
5711 : {
5712 16944 : SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
5713 16944 : vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
5714 : slp_node, cost_vec);
5715 : }
5716 68683 : else if (modifier == NARROW_SRC || modifier == NARROW_DST)
5717 : {
5718 28084 : SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
5719 : /* The final packing step produces one vector result per copy. */
5720 28084 : vect_model_promotion_demotion_cost (slp_node, nvectors,
5721 : multi_step_cvt, cost_vec,
5722 : widen_arith);
5723 : }
5724 : else
5725 : {
5726 40599 : SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
5727 : /* The initial unpacking step produces two vector results
5728 : per copy. MULTI_STEP_CVT is 0 for a single conversion,
5729 : so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5730 40599 : vect_model_promotion_demotion_cost (slp_node,
5731 : nvectors >> multi_step_cvt,
5732 : multi_step_cvt, cost_vec,
5733 : widen_arith);
5734 : }
5735 85627 : interm_types.release ();
5736 85627 : return true;
5737 : }
5738 :
5739 : /* Transform. */
5740 23983 : if (dump_enabled_p ())
5741 4258 : dump_printf_loc (MSG_NOTE, vect_location, "transform conversion.\n");
5742 :
5743 : /* In case of multi-step conversion, we first generate conversion operations
5744 : to the intermediate types, and then from that types to the final one.
5745 : We create vector destinations for the intermediate type (TYPES) received
5746 : from supportable_*_operation, and store them in the correct order
5747 : for future use in vect_create_vectorized_*_stmts (). */
5748 47966 : auto_vec<tree> vec_dsts (multi_step_cvt + 1);
5749 23983 : bool widen_or_narrow_float_p
5750 23983 : = cvt_type && (modifier == WIDEN || modifier == NARROW_SRC);
5751 23983 : vec_dest = vect_create_destination_var (scalar_dest,
5752 : widen_or_narrow_float_p
5753 : ? cvt_type : vectype_out);
5754 23983 : vec_dsts.quick_push (vec_dest);
5755 :
5756 23983 : if (multi_step_cvt)
5757 : {
5758 9252 : for (i = interm_types.length () - 1;
5759 9252 : interm_types.iterate (i, &intermediate_type); i--)
5760 : {
5761 4869 : vec_dest = vect_create_destination_var (scalar_dest,
5762 : intermediate_type);
5763 4869 : vec_dsts.quick_push (vec_dest);
5764 : }
5765 : }
5766 :
5767 23983 : if (cvt_type)
5768 76 : vec_dest = vect_create_destination_var (scalar_dest,
5769 : widen_or_narrow_float_p
5770 : ? vectype_out : cvt_type);
5771 :
5772 23983 : unsigned num_vectors = vect_get_num_copies (vinfo, slp_node);
5773 23983 : switch (modifier)
5774 : {
5775 4756 : case NONE:
5776 4756 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5777 : /* vec_dest is intermediate type operand when multi_step_cvt. */
5778 4756 : if (multi_step_cvt)
5779 : {
5780 21 : cvt_op = vec_dest;
5781 21 : vec_dest = vec_dsts[0];
5782 : }
5783 :
5784 9937 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5785 : {
5786 : /* Arguments are ready, create the new vector stmt. */
5787 5181 : gimple* new_stmt;
5788 5181 : if (multi_step_cvt)
5789 : {
5790 21 : gcc_assert (multi_step_cvt == 1);
5791 21 : new_stmt = vect_gimple_build (cvt_op, codecvt1, vop0);
5792 21 : new_temp = make_ssa_name (cvt_op, new_stmt);
5793 21 : gimple_assign_set_lhs (new_stmt, new_temp);
5794 21 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5795 21 : vop0 = new_temp;
5796 : }
5797 5181 : if (code1 == COND_EXPR)
5798 : {
5799 1 : gcc_assert (!multi_step_cvt);
5800 1 : new_stmt
5801 2 : = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
5802 : build_minus_one_cst
5803 1 : (TREE_TYPE (vec_dest)),
5804 1 : build_zero_cst (TREE_TYPE (vec_dest)));
5805 1 : new_temp = make_ssa_name (vec_dest, new_stmt);
5806 1 : gimple_set_lhs (new_stmt, new_temp);
5807 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5808 1 : tree new_temp2 = make_ssa_name (vectype_out);
5809 1 : new_stmt = gimple_build_assign (new_temp2,
5810 : build1 (VIEW_CONVERT_EXPR,
5811 : vectype_out, new_temp));
5812 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5813 1 : slp_node->push_vec_def (new_temp2);
5814 : }
5815 : else
5816 : {
5817 5180 : new_stmt = vect_gimple_build (vec_dest, code1, vop0);
5818 5180 : new_temp = make_ssa_name (vec_dest, new_stmt);
5819 5180 : gimple_set_lhs (new_stmt, new_temp);
5820 5180 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5821 :
5822 5180 : slp_node->push_vec_def (new_stmt);
5823 : }
5824 : }
5825 : break;
5826 :
5827 9821 : case WIDEN:
5828 : /* In case the vectorization factor (VF) is bigger than the number
5829 : of elements that we can fit in a vectype (nunits), we have to
5830 : generate more than one vector stmt - i.e - we need to "unroll"
5831 : the vector stmt by a factor VF/nunits. */
5832 19642 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
5833 9821 : code != WIDEN_LSHIFT_EXPR && slp_op1, &vec_oprnds1);
5834 9821 : if (code == WIDEN_LSHIFT_EXPR)
5835 : {
5836 0 : int oprnds_size = vec_oprnds0.length ();
5837 0 : vec_oprnds1.create (oprnds_size);
5838 0 : for (i = 0; i < oprnds_size; ++i)
5839 0 : vec_oprnds1.quick_push (vect_get_slp_scalar_def (slp_op1, 0));
5840 : }
5841 : /* Arguments are ready. Create the new vector stmts. */
5842 21466 : for (i = multi_step_cvt; i >= 0; i--)
5843 : {
5844 11645 : tree this_dest = vec_dsts[i];
5845 11645 : code_helper c1 = code1, c2 = code2;
5846 11645 : if (i == 0 && codecvt2 != ERROR_MARK)
5847 : {
5848 48 : c1 = codecvt1;
5849 48 : c2 = codecvt2;
5850 : }
5851 11645 : if (known_eq (nunits_out, nunits_in))
5852 44 : vect_create_half_widening_stmts (vinfo, &vec_oprnds0, &vec_oprnds1,
5853 : stmt_info, this_dest, gsi, c1,
5854 : op_type);
5855 : else
5856 : /* ??? For constant/external inputs we can end up with
5857 : excess lanes. When the number of inputs already match
5858 : the number of required outputs request half of the
5859 : lanes (gcc.dg/vect/O3-vect-pr32243.c). Low coverage
5860 : makes this likely incomplete. */
5861 13425 : vect_create_vectorized_promotion_stmts (vinfo, &vec_oprnds0,
5862 : &vec_oprnds1, stmt_info,
5863 : this_dest, gsi,
5864 : c1, c2, op_type,
5865 : i == 0 ? num_vectors : -1u);
5866 : }
5867 :
5868 61511 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5869 : {
5870 27707 : gimple *new_stmt;
5871 27707 : if (cvt_type)
5872 : {
5873 120 : new_temp = make_ssa_name (vec_dest);
5874 120 : new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5875 120 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5876 : }
5877 : else
5878 27587 : new_stmt = SSA_NAME_DEF_STMT (vop0);
5879 :
5880 27707 : slp_node->push_vec_def (new_stmt);
5881 : }
5882 : break;
5883 :
5884 9406 : case NARROW_SRC:
5885 9406 : case NARROW_DST:
5886 : /* In case the vectorization factor (VF) is bigger than the number
5887 : of elements that we can fit in a vectype (nunits), we have to
5888 : generate more than one vector stmt - i.e - we need to "unroll"
5889 : the vector stmt by a factor VF/nunits. */
5890 9406 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0);
5891 : /* Arguments are ready. Create the new vector stmts. */
5892 9406 : if (cvt_type && modifier == NARROW_DST)
5893 9559 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5894 : {
5895 124 : new_temp = make_ssa_name (vec_dest);
5896 124 : gimple *new_stmt = vect_gimple_build (new_temp, codecvt1, vop0);
5897 124 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5898 124 : vec_oprnds0[i] = new_temp;
5899 : }
5900 :
5901 9406 : vect_create_vectorized_demotion_stmts (vinfo, &vec_oprnds0,
5902 : multi_step_cvt,
5903 : stmt_info, vec_dsts, gsi,
5904 : slp_node, code1,
5905 : modifier == NARROW_SRC);
5906 : /* After demoting slp_op0 to cvt_type, convert it to dest. */
5907 9406 : if (cvt_type && code == FLOAT_EXPR)
5908 : {
5909 32 : for (unsigned int i = 0; i != vec_oprnds0.length() / 2; i++)
5910 : {
5911 : /* Arguments are ready, create the new vector stmt. */
5912 8 : gcc_assert (TREE_CODE_LENGTH ((tree_code) codecvt1) == unary_op);
5913 8 : gimple *new_stmt
5914 8 : = vect_gimple_build (vec_dest, codecvt1, vec_oprnds0[i]);
5915 8 : new_temp = make_ssa_name (vec_dest, new_stmt);
5916 8 : gimple_set_lhs (new_stmt, new_temp);
5917 8 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
5918 :
5919 : /* This is the last step of the conversion sequence. Store the
5920 : vectors in SLP_NODE or in vector info of the scalar statement
5921 : (or in STMT_VINFO_RELATED_STMT chain). */
5922 8 : slp_node->push_vec_def (new_stmt);
5923 : }
5924 : }
5925 : break;
5926 : }
5927 :
5928 23983 : vec_oprnds0.release ();
5929 23983 : vec_oprnds1.release ();
5930 23983 : interm_types.release ();
5931 :
5932 23983 : return true;
5933 192568 : }
5934 :
5935 : /* Return true if we can assume from the scalar form of STMT_INFO that
5936 : neither the scalar nor the vector forms will generate code. STMT_INFO
5937 : is known not to involve a data reference. */
5938 :
5939 : bool
5940 3296148 : vect_nop_conversion_p (stmt_vec_info stmt_info)
5941 : {
5942 3296148 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5943 2997083 : if (!stmt || STMT_VINFO_DATA_REF (stmt_info))
5944 : return false;
5945 :
5946 984194 : tree lhs = gimple_assign_lhs (stmt);
5947 984194 : tree_code code = gimple_assign_rhs_code (stmt);
5948 984194 : tree rhs = gimple_assign_rhs1 (stmt);
5949 :
5950 984194 : if (code == SSA_NAME || code == VIEW_CONVERT_EXPR)
5951 : return true;
5952 :
5953 980944 : if (CONVERT_EXPR_CODE_P (code))
5954 245113 : return tree_nop_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
5955 :
5956 : return false;
5957 : }
5958 :
5959 : /* Function vectorizable_assignment.
5960 :
5961 : Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5962 : If COST_VEC is passed, calculate costs but don't change anything,
5963 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
5964 : it, and insert it at GSI.
5965 : Return true if STMT_INFO is vectorizable in this way. */
5966 :
5967 : static bool
5968 2139513 : vectorizable_assignment (vec_info *vinfo,
5969 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
5970 : slp_tree slp_node,
5971 : stmt_vector_for_cost *cost_vec)
5972 : {
5973 2139513 : tree vec_dest;
5974 2139513 : tree scalar_dest;
5975 2139513 : tree op;
5976 2139513 : tree new_temp;
5977 2139513 : enum vect_def_type dt[1] = {vect_unknown_def_type};
5978 2139513 : int i;
5979 2139513 : vec<tree> vec_oprnds = vNULL;
5980 2139513 : tree vop;
5981 2139513 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5982 2139513 : enum tree_code code;
5983 2139513 : tree vectype_in;
5984 :
5985 2139513 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5986 : return false;
5987 :
5988 2139513 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5989 244891 : && cost_vec)
5990 : return false;
5991 :
5992 : /* Is vectorizable assignment? */
5993 1894622 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
5994 1818911 : if (!stmt)
5995 : return false;
5996 :
5997 1818911 : scalar_dest = gimple_assign_lhs (stmt);
5998 1818911 : if (TREE_CODE (scalar_dest) != SSA_NAME)
5999 : return false;
6000 :
6001 989038 : if (STMT_VINFO_DATA_REF (stmt_info))
6002 : return false;
6003 :
6004 417517 : code = gimple_assign_rhs_code (stmt);
6005 417517 : if (!(gimple_assign_single_p (stmt)
6006 415891 : || code == PAREN_EXPR
6007 414619 : || CONVERT_EXPR_CODE_P (code)))
6008 : return false;
6009 :
6010 104722 : tree vectype = SLP_TREE_VECTYPE (slp_node);
6011 104722 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6012 :
6013 104722 : slp_tree slp_op;
6014 104722 : if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
6015 : &dt[0], &vectype_in))
6016 : {
6017 0 : if (dump_enabled_p ())
6018 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6019 : "use not simple.\n");
6020 : return false;
6021 : }
6022 104722 : if (!vectype_in)
6023 19575 : vectype_in = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), slp_node);
6024 :
6025 : /* We can handle VIEW_CONVERT conversions that do not change the number
6026 : of elements or the vector size or other conversions when the component
6027 : types are nop-convertible. */
6028 104722 : if (!vectype_in
6029 109358 : || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
6030 92379 : || (code == VIEW_CONVERT_EXPR
6031 2996 : && maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
6032 2996 : GET_MODE_SIZE (TYPE_MODE (vectype_in))))
6033 197101 : || (CONVERT_EXPR_CODE_P (code)
6034 89513 : && !tree_nop_conversion_p (TREE_TYPE (vectype),
6035 89513 : TREE_TYPE (vectype_in))))
6036 : return false;
6037 :
6038 262776 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != VECTOR_BOOLEAN_TYPE_P (vectype_in))
6039 : {
6040 27 : if (dump_enabled_p ())
6041 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6042 : "can't convert between boolean and non "
6043 0 : "boolean vectors %T\n", TREE_TYPE (op));
6044 :
6045 : return false;
6046 : }
6047 :
6048 : /* We do not handle bit-precision changes. */
6049 87716 : if ((CONVERT_EXPR_CODE_P (code)
6050 2866 : || code == VIEW_CONVERT_EXPR)
6051 86348 : && ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6052 85103 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6053 85850 : || (INTEGRAL_TYPE_P (TREE_TYPE (op))
6054 77037 : && !type_has_mode_precision_p (TREE_TYPE (op))))
6055 : /* But a conversion that does not change the bit-pattern is ok. */
6056 88778 : && !(INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
6057 1062 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
6058 1062 : && (((TYPE_PRECISION (TREE_TYPE (scalar_dest))
6059 1062 : > TYPE_PRECISION (TREE_TYPE (op)))
6060 564 : && TYPE_UNSIGNED (TREE_TYPE (op)))
6061 560 : || (TYPE_PRECISION (TREE_TYPE (scalar_dest))
6062 560 : == TYPE_PRECISION (TREE_TYPE (op))))))
6063 : {
6064 329 : if (dump_enabled_p ())
6065 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6066 : "type conversion to/from bit-precision "
6067 : "unsupported.\n");
6068 : return false;
6069 : }
6070 :
6071 87387 : if (cost_vec) /* transformation not required. */
6072 : {
6073 70409 : if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
6074 : {
6075 0 : if (dump_enabled_p ())
6076 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6077 : "incompatible vector types for invariants\n");
6078 : return false;
6079 : }
6080 70409 : SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
6081 70409 : DUMP_VECT_SCOPE ("vectorizable_assignment");
6082 70409 : if (!vect_nop_conversion_p (stmt_info))
6083 1082 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6084 70409 : return true;
6085 : }
6086 :
6087 : /* Transform. */
6088 16978 : if (dump_enabled_p ())
6089 3610 : dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
6090 :
6091 : /* Handle def. */
6092 16978 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6093 :
6094 : /* Handle use. */
6095 16978 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds);
6096 :
6097 : /* Arguments are ready. create the new vector stmt. */
6098 55427 : FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
6099 : {
6100 21471 : gassign *new_stmt;
6101 21471 : if (code == PAREN_EXPR)
6102 510 : new_stmt = gimple_build_assign (vec_dest, PAREN_EXPR, vop);
6103 : else
6104 : {
6105 20961 : if (CONVERT_EXPR_CODE_P (code)
6106 214 : || code == VIEW_CONVERT_EXPR)
6107 20905 : vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
6108 20961 : new_stmt = gimple_build_assign (vec_dest, vop);
6109 : }
6110 21471 : new_temp = make_ssa_name (vec_dest, new_stmt);
6111 21471 : gimple_assign_set_lhs (new_stmt, new_temp);
6112 21471 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6113 21471 : slp_node->push_vec_def (new_stmt);
6114 : }
6115 :
6116 16978 : vec_oprnds.release ();
6117 16978 : return true;
6118 : }
6119 :
6120 :
6121 : /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6122 : either as shift by a scalar or by a vector. */
6123 :
6124 : bool
6125 306375 : vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
6126 : {
6127 306375 : optab optab;
6128 306375 : tree vectype;
6129 :
6130 306375 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6131 306375 : if (!vectype)
6132 : return false;
6133 :
6134 306375 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6135 306375 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6136 : return true;
6137 :
6138 269205 : optab = optab_for_tree_code (code, vectype, optab_vector);
6139 269205 : if (optab && can_implement_p (optab, TYPE_MODE (vectype)))
6140 : return true;
6141 :
6142 : return false;
6143 : }
6144 :
6145 :
6146 : /* Function vectorizable_shift.
6147 :
6148 : Check if STMT_INFO performs a shift operation that can be vectorized.
6149 : If COST_VEC is passed, calculate costs but don't change anything,
6150 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6151 : it, and insert it at GSI.
6152 : Return true if STMT_INFO is vectorizable in this way. */
6153 :
6154 : static bool
6155 771894 : vectorizable_shift (vec_info *vinfo,
6156 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6157 : slp_tree slp_node,
6158 : stmt_vector_for_cost *cost_vec)
6159 : {
6160 771894 : tree vec_dest;
6161 771894 : tree scalar_dest;
6162 771894 : tree op0, op1 = NULL;
6163 771894 : tree vec_oprnd1 = NULL_TREE;
6164 771894 : tree vectype;
6165 771894 : enum tree_code code;
6166 771894 : machine_mode vec_mode;
6167 771894 : tree new_temp;
6168 771894 : optab optab;
6169 771894 : int icode;
6170 771894 : machine_mode optab_op2_mode;
6171 771894 : enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
6172 771894 : poly_uint64 nunits_in;
6173 771894 : poly_uint64 nunits_out;
6174 771894 : tree vectype_out;
6175 771894 : tree op1_vectype;
6176 771894 : int i;
6177 771894 : vec<tree> vec_oprnds0 = vNULL;
6178 771894 : vec<tree> vec_oprnds1 = vNULL;
6179 771894 : tree vop0, vop1;
6180 771894 : unsigned int k;
6181 771894 : bool scalar_shift_arg = true;
6182 771894 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6183 771894 : bool incompatible_op1_vectype_p = false;
6184 :
6185 771894 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6186 : return false;
6187 :
6188 771894 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6189 244891 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
6190 243512 : && cost_vec)
6191 : return false;
6192 :
6193 : /* Is STMT a vectorizable binary/unary operation? */
6194 528382 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6195 453761 : if (!stmt)
6196 : return false;
6197 :
6198 453761 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
6199 : return false;
6200 :
6201 453195 : code = gimple_assign_rhs_code (stmt);
6202 :
6203 453195 : if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
6204 : || code == RROTATE_EXPR))
6205 : return false;
6206 :
6207 59799 : scalar_dest = gimple_assign_lhs (stmt);
6208 59799 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6209 59799 : if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
6210 : {
6211 0 : if (dump_enabled_p ())
6212 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6213 : "bit-precision shifts not supported.\n");
6214 : return false;
6215 : }
6216 :
6217 59799 : slp_tree slp_op0;
6218 59799 : if (!vect_is_simple_use (vinfo, slp_node,
6219 : 0, &op0, &slp_op0, &dt[0], &vectype))
6220 : {
6221 0 : if (dump_enabled_p ())
6222 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6223 : "use not simple.\n");
6224 : return false;
6225 : }
6226 : /* If op0 is an external or constant def, infer the vector type
6227 : from the scalar type. */
6228 59799 : if (!vectype)
6229 14557 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
6230 59799 : if (!cost_vec)
6231 8968 : gcc_assert (vectype);
6232 59799 : if (!vectype)
6233 : {
6234 0 : if (dump_enabled_p ())
6235 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6236 : "no vectype for scalar type\n");
6237 : return false;
6238 : }
6239 :
6240 59799 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6241 59799 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6242 59799 : if (maybe_ne (nunits_out, nunits_in))
6243 : return false;
6244 :
6245 59799 : stmt_vec_info op1_def_stmt_info;
6246 59799 : slp_tree slp_op1;
6247 59799 : if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
6248 : &dt[1], &op1_vectype, &op1_def_stmt_info))
6249 : {
6250 0 : if (dump_enabled_p ())
6251 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6252 : "use not simple.\n");
6253 : return false;
6254 : }
6255 :
6256 : /* Determine whether the shift amount is a vector, or scalar. If the
6257 : shift/rotate amount is a vector, use the vector/vector shift optabs. */
6258 :
6259 59799 : if ((dt[1] == vect_internal_def
6260 59799 : || dt[1] == vect_induction_def
6261 43158 : || dt[1] == vect_nested_cycle)
6262 16659 : && SLP_TREE_LANES (slp_node) == 1)
6263 : scalar_shift_arg = false;
6264 43195 : else if (dt[1] == vect_constant_def
6265 : || dt[1] == vect_external_def
6266 43195 : || dt[1] == vect_internal_def)
6267 : {
6268 : /* In SLP, need to check whether the shift count is the same,
6269 : in loops if it is a constant or invariant, it is always
6270 : a scalar shift. */
6271 43189 : vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
6272 43189 : stmt_vec_info slpstmt_info;
6273 :
6274 112358 : FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
6275 69169 : if (slpstmt_info)
6276 : {
6277 69169 : gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
6278 138338 : if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
6279 69169 : scalar_shift_arg = false;
6280 : }
6281 :
6282 : /* For internal SLP defs we have to make sure we see scalar stmts
6283 : for all vector elements.
6284 : ??? For different vectors we could resort to a different
6285 : scalar shift operand but code-generation below simply always
6286 : takes the first. */
6287 43189 : if (dt[1] == vect_internal_def
6288 43238 : && maybe_ne (nunits_out * vect_get_num_copies (vinfo, slp_node),
6289 49 : stmts.length ()))
6290 : scalar_shift_arg = false;
6291 :
6292 : /* If the shift amount is computed by a pattern stmt we cannot
6293 : use the scalar amount directly thus give up and use a vector
6294 : shift. */
6295 43189 : if (op1_def_stmt_info && is_pattern_stmt_p (op1_def_stmt_info))
6296 : scalar_shift_arg = false;
6297 : }
6298 : else
6299 : {
6300 6 : if (dump_enabled_p ())
6301 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6302 : "operand mode requires invariant argument.\n");
6303 : return false;
6304 : }
6305 :
6306 : /* Vector shifted by vector. */
6307 59831 : bool was_scalar_shift_arg = scalar_shift_arg;
6308 43180 : if (!scalar_shift_arg)
6309 : {
6310 16651 : optab = optab_for_tree_code (code, vectype, optab_vector);
6311 16651 : if (dump_enabled_p ())
6312 1225 : dump_printf_loc (MSG_NOTE, vect_location,
6313 : "vector/vector shift/rotate found.\n");
6314 :
6315 16651 : if (!op1_vectype)
6316 15 : op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1),
6317 : slp_op1);
6318 16651 : incompatible_op1_vectype_p
6319 33302 : = (op1_vectype == NULL_TREE
6320 16651 : || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype),
6321 16651 : TYPE_VECTOR_SUBPARTS (vectype))
6322 33300 : || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
6323 16651 : if (incompatible_op1_vectype_p
6324 7 : && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
6325 1 : || slp_op1->refcnt != 1))
6326 : {
6327 6 : if (dump_enabled_p ())
6328 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6329 : "unusable type for last operand in"
6330 : " vector/vector shift/rotate.\n");
6331 : return false;
6332 : }
6333 : }
6334 : /* See if the machine has a vector shifted by scalar insn and if not
6335 : then see if it has a vector shifted by vector insn. */
6336 : else
6337 : {
6338 43142 : optab = optab_for_tree_code (code, vectype, optab_scalar);
6339 43142 : if (optab
6340 43142 : && can_implement_p (optab, TYPE_MODE (vectype)))
6341 : {
6342 42855 : if (dump_enabled_p ())
6343 5178 : dump_printf_loc (MSG_NOTE, vect_location,
6344 : "vector/scalar shift/rotate found.\n");
6345 : }
6346 : else
6347 : {
6348 287 : optab = optab_for_tree_code (code, vectype, optab_vector);
6349 287 : if (optab
6350 287 : && can_implement_p (optab, TYPE_MODE (vectype)))
6351 : {
6352 0 : scalar_shift_arg = false;
6353 :
6354 0 : if (dump_enabled_p ())
6355 0 : dump_printf_loc (MSG_NOTE, vect_location,
6356 : "vector/vector shift/rotate found.\n");
6357 :
6358 0 : if (!op1_vectype)
6359 0 : op1_vectype = get_vectype_for_scalar_type (vinfo,
6360 0 : TREE_TYPE (op1),
6361 : slp_op1);
6362 :
6363 : /* Unlike the other binary operators, shifts/rotates have
6364 : the rhs being int, instead of the same type as the lhs,
6365 : so make sure the scalar is the right type if we are
6366 : dealing with vectors of long long/long/short/char. */
6367 0 : incompatible_op1_vectype_p
6368 0 : = (!op1_vectype
6369 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype),
6370 0 : TREE_TYPE (op1)));
6371 : if (incompatible_op1_vectype_p
6372 0 : && dt[1] == vect_internal_def)
6373 : {
6374 0 : if (dump_enabled_p ())
6375 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6376 : "unusable type for last operand in"
6377 : " vector/vector shift/rotate.\n");
6378 : return false;
6379 : }
6380 : }
6381 : }
6382 : }
6383 :
6384 : /* Supportable by target? */
6385 59787 : if (!optab)
6386 : {
6387 0 : if (dump_enabled_p ())
6388 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6389 : "no shift optab for %s and %T.\n",
6390 : get_tree_code_name (code), vectype);
6391 : return false;
6392 : }
6393 59787 : vec_mode = TYPE_MODE (vectype);
6394 59787 : icode = (int) optab_handler (optab, vec_mode);
6395 59787 : if (icode == CODE_FOR_nothing)
6396 : {
6397 6579 : if (dump_enabled_p ())
6398 948 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6399 : "shift op not supported by target.\n");
6400 : return false;
6401 : }
6402 : /* vector lowering cannot optimize vector shifts using word arithmetic. */
6403 53208 : if (vect_emulated_vector_p (vectype))
6404 : return false;
6405 :
6406 53208 : if (cost_vec) /* transformation not required. */
6407 : {
6408 44240 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6409 44240 : || ((!scalar_shift_arg || dt[1] == vect_internal_def)
6410 8201 : && (!incompatible_op1_vectype_p
6411 1 : || dt[1] == vect_constant_def)
6412 8201 : && !vect_maybe_update_slp_op_vectype
6413 8201 : (slp_op1,
6414 : incompatible_op1_vectype_p ? vectype : op1_vectype)))
6415 : {
6416 0 : if (dump_enabled_p ())
6417 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6418 : "incompatible vector types for invariants\n");
6419 : return false;
6420 : }
6421 : /* Now adjust the constant shift amount in place. */
6422 44240 : if (incompatible_op1_vectype_p
6423 1 : && dt[1] == vect_constant_def)
6424 : {
6425 1 : unsigned group_size = SLP_TREE_LANES (slp_op1);
6426 2 : gcc_assert (SLP_TREE_SCALAR_OPS (slp_op1).length () == group_size);
6427 5 : for (unsigned i = 0; i < group_size; ++i)
6428 : {
6429 4 : SLP_TREE_SCALAR_OPS (slp_op1)[i]
6430 4 : = fold_convert (TREE_TYPE (vectype),
6431 : SLP_TREE_SCALAR_OPS (slp_op1)[i]);
6432 4 : gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
6433 : == INTEGER_CST));
6434 : }
6435 : }
6436 44240 : SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
6437 44240 : DUMP_VECT_SCOPE ("vectorizable_shift");
6438 44240 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6439 44240 : return true;
6440 : }
6441 :
6442 : /* Transform. */
6443 :
6444 8968 : if (dump_enabled_p ())
6445 2118 : dump_printf_loc (MSG_NOTE, vect_location,
6446 : "transform binary/unary operation.\n");
6447 :
6448 : /* Handle def. */
6449 8968 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6450 :
6451 8968 : unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
6452 8968 : if (scalar_shift_arg && dt[1] != vect_internal_def)
6453 : {
6454 : /* Vector shl and shr insn patterns can be defined with scalar
6455 : operand 2 (shift operand). In this case, use constant or loop
6456 : invariant op1 directly, without extending it to vector mode
6457 : first. */
6458 6796 : optab_op2_mode = insn_data[icode].operand[2].mode;
6459 6796 : if (!VECTOR_MODE_P (optab_op2_mode))
6460 : {
6461 6796 : if (dump_enabled_p ())
6462 2003 : dump_printf_loc (MSG_NOTE, vect_location,
6463 : "operand 1 using scalar mode.\n");
6464 6796 : vec_oprnd1 = op1;
6465 6796 : vec_oprnds1.create (nvectors);
6466 6796 : vec_oprnds1.quick_push (vec_oprnd1);
6467 : /* Store vec_oprnd1 for every vector stmt to be created.
6468 : We check during the analysis that all the shift arguments
6469 : are the same.
6470 : TODO: Allow different constants for different vector
6471 : stmts generated for an SLP instance. */
6472 15582 : for (k = 0; k < nvectors - 1; k++)
6473 1990 : vec_oprnds1.quick_push (vec_oprnd1);
6474 : }
6475 : }
6476 2172 : else if (!scalar_shift_arg && incompatible_op1_vectype_p)
6477 : {
6478 0 : if (was_scalar_shift_arg)
6479 : {
6480 : /* If the argument was the same in all lanes create the
6481 : correctly typed vector shift amount directly. Note
6482 : we made SLP scheduling think we use the original scalars,
6483 : so place the compensation code next to the shift which
6484 : is conservative. See PR119640 where it otherwise breaks. */
6485 0 : op1 = fold_convert (TREE_TYPE (vectype), op1);
6486 0 : op1 = vect_init_vector (vinfo, stmt_info, op1, TREE_TYPE (vectype),
6487 : gsi);
6488 0 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, op1, vectype,
6489 : gsi);
6490 0 : vec_oprnds1.create (nvectors);
6491 0 : for (k = 0; k < nvectors; k++)
6492 0 : vec_oprnds1.quick_push (vec_oprnd1);
6493 : }
6494 0 : else if (dt[1] == vect_constant_def)
6495 : /* The constant shift amount has been adjusted in place. */
6496 : ;
6497 : else
6498 0 : gcc_assert (TYPE_MODE (op1_vectype) == TYPE_MODE (vectype));
6499 : }
6500 :
6501 : /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6502 : (a special case for certain kind of vector shifts); otherwise,
6503 : operand 1 should be of a vector type (the usual case). */
6504 8968 : vect_get_vec_defs (vinfo, slp_node,
6505 : true, &vec_oprnds0, !vec_oprnd1, &vec_oprnds1);
6506 :
6507 : /* Arguments are ready. Create the new vector stmt. */
6508 32404 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6509 : {
6510 : /* For internal defs where we need to use a scalar shift arg
6511 : extract the first lane. */
6512 14468 : if (scalar_shift_arg && dt[1] == vect_internal_def)
6513 : {
6514 10 : vop1 = vec_oprnds1[0];
6515 10 : new_temp = make_ssa_name (TREE_TYPE (TREE_TYPE (vop1)));
6516 10 : gassign *new_stmt
6517 10 : = gimple_build_assign (new_temp,
6518 10 : build3 (BIT_FIELD_REF, TREE_TYPE (new_temp),
6519 : vop1,
6520 10 : TYPE_SIZE (TREE_TYPE (new_temp)),
6521 : bitsize_zero_node));
6522 10 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6523 10 : vop1 = new_temp;
6524 10 : }
6525 : else
6526 14458 : vop1 = vec_oprnds1[i];
6527 14468 : gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
6528 14468 : new_temp = make_ssa_name (vec_dest, new_stmt);
6529 14468 : gimple_assign_set_lhs (new_stmt, new_temp);
6530 14468 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6531 14468 : slp_node->push_vec_def (new_stmt);
6532 : }
6533 :
6534 8968 : vec_oprnds0.release ();
6535 8968 : vec_oprnds1.release ();
6536 :
6537 8968 : return true;
6538 : }
6539 :
6540 : /* Function vectorizable_operation.
6541 :
6542 : Check if STMT_INFO performs a binary, unary or ternary operation that can
6543 : be vectorized.
6544 : If COST_VEC is passed, calculate costs but don't change anything,
6545 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
6546 : it, and insert it at GSI.
6547 : Return true if STMT_INFO is vectorizable in this way. */
6548 :
6549 : static bool
6550 2664194 : vectorizable_operation (vec_info *vinfo,
6551 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
6552 : slp_tree slp_node,
6553 : stmt_vector_for_cost *cost_vec)
6554 : {
6555 2664194 : tree vec_dest;
6556 2664194 : tree scalar_dest;
6557 2664194 : tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
6558 2664194 : tree vectype;
6559 2664194 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6560 2664194 : enum tree_code code, orig_code;
6561 2664194 : machine_mode vec_mode;
6562 2664194 : tree new_temp;
6563 2664194 : int op_type;
6564 2664194 : optab optab;
6565 2664194 : bool target_support_p;
6566 2664194 : enum vect_def_type dt[3]
6567 : = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
6568 2664194 : poly_uint64 nunits_in;
6569 2664194 : poly_uint64 nunits_out;
6570 2664194 : tree vectype_out;
6571 2664194 : int i;
6572 2664194 : vec<tree> vec_oprnds0 = vNULL;
6573 2664194 : vec<tree> vec_oprnds1 = vNULL;
6574 2664194 : vec<tree> vec_oprnds2 = vNULL;
6575 2664194 : tree vop0, vop1, vop2;
6576 2664194 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
6577 :
6578 2664194 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6579 : return false;
6580 :
6581 2664194 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6582 244891 : && cost_vec)
6583 : return false;
6584 :
6585 : /* Is STMT a vectorizable binary/unary operation? */
6586 2419303 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
6587 2343592 : if (!stmt)
6588 : return false;
6589 :
6590 : /* Loads and stores are handled in vectorizable_{load,store}. */
6591 2343592 : if (STMT_VINFO_DATA_REF (stmt_info))
6592 : return false;
6593 :
6594 942198 : orig_code = code = gimple_assign_rhs_code (stmt);
6595 :
6596 : /* Shifts are handled in vectorizable_shift. */
6597 942198 : if (code == LSHIFT_EXPR
6598 : || code == RSHIFT_EXPR
6599 : || code == LROTATE_EXPR
6600 942198 : || code == RROTATE_EXPR)
6601 : return false;
6602 :
6603 : /* Comparisons are handled in vectorizable_comparison. */
6604 891367 : if (TREE_CODE_CLASS (code) == tcc_comparison)
6605 : return false;
6606 :
6607 : /* Conditions are handled in vectorizable_condition. */
6608 691014 : if (code == COND_EXPR)
6609 : return false;
6610 :
6611 : /* For pointer addition and subtraction, we should use the normal
6612 : plus and minus for the vector operation. */
6613 663949 : if (code == POINTER_PLUS_EXPR)
6614 : code = PLUS_EXPR;
6615 643799 : if (code == POINTER_DIFF_EXPR)
6616 3027 : code = MINUS_EXPR;
6617 :
6618 : /* Support only unary or binary operations. */
6619 663949 : op_type = TREE_CODE_LENGTH (code);
6620 663949 : if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
6621 : {
6622 0 : if (dump_enabled_p ())
6623 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6624 : "num. args = %d (not unary/binary/ternary op).\n",
6625 : op_type);
6626 : return false;
6627 : }
6628 :
6629 663949 : scalar_dest = gimple_assign_lhs (stmt);
6630 663949 : vectype_out = SLP_TREE_VECTYPE (slp_node);
6631 :
6632 : /* Most operations cannot handle bit-precision types without extra
6633 : truncations. */
6634 663949 : bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
6635 652416 : if (!mask_op_p
6636 652416 : && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
6637 : /* Exception are bitwise binary operations. */
6638 : && code != BIT_IOR_EXPR
6639 2022 : && code != BIT_XOR_EXPR
6640 1146 : && code != BIT_AND_EXPR)
6641 : {
6642 814 : if (dump_enabled_p ())
6643 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6644 : "bit-precision arithmetic not supported.\n");
6645 : return false;
6646 : }
6647 :
6648 663135 : slp_tree slp_op0;
6649 663135 : if (!vect_is_simple_use (vinfo, slp_node,
6650 : 0, &op0, &slp_op0, &dt[0], &vectype))
6651 : {
6652 0 : if (dump_enabled_p ())
6653 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6654 : "use not simple.\n");
6655 : return false;
6656 : }
6657 663135 : bool is_invariant = (dt[0] == vect_external_def
6658 663135 : || dt[0] == vect_constant_def);
6659 : /* If op0 is an external or constant def, infer the vector type
6660 : from the scalar type. */
6661 663135 : if (!vectype)
6662 : {
6663 : /* For boolean type we cannot determine vectype by
6664 : invariant value (don't know whether it is a vector
6665 : of booleans or vector of integers). We use output
6666 : vectype because operations on boolean don't change
6667 : type. */
6668 78148 : if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
6669 : {
6670 2154 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
6671 : {
6672 317 : if (dump_enabled_p ())
6673 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6674 : "not supported operation on bool value.\n");
6675 : return false;
6676 : }
6677 1837 : vectype = vectype_out;
6678 : }
6679 : else
6680 75994 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
6681 : slp_node);
6682 : }
6683 662818 : if (!cost_vec)
6684 117658 : gcc_assert (vectype);
6685 662818 : if (!vectype)
6686 : {
6687 311 : if (dump_enabled_p ())
6688 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6689 : "no vectype for scalar type %T\n",
6690 2 : TREE_TYPE (op0));
6691 :
6692 : return false;
6693 : }
6694 :
6695 662507 : nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
6696 662507 : nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
6697 662507 : if (maybe_ne (nunits_out, nunits_in)
6698 662507 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out), TREE_TYPE (vectype)))
6699 : return false;
6700 :
6701 641650 : tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
6702 641650 : slp_tree slp_op1 = NULL, slp_op2 = NULL;
6703 641650 : if (op_type == binary_op || op_type == ternary_op)
6704 : {
6705 560225 : if (!vect_is_simple_use (vinfo, slp_node,
6706 : 1, &op1, &slp_op1, &dt[1], &vectype2))
6707 : {
6708 0 : if (dump_enabled_p ())
6709 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6710 : "use not simple.\n");
6711 : return false;
6712 : }
6713 560225 : is_invariant &= (dt[1] == vect_external_def
6714 560225 : || dt[1] == vect_constant_def);
6715 560225 : if (vectype2
6716 907791 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype2))
6717 347566 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6718 347566 : TREE_TYPE (vectype2))))
6719 : return false;
6720 : }
6721 641650 : if (op_type == ternary_op)
6722 : {
6723 0 : if (!vect_is_simple_use (vinfo, slp_node,
6724 : 2, &op2, &slp_op2, &dt[2], &vectype3))
6725 : {
6726 0 : if (dump_enabled_p ())
6727 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6728 : "use not simple.\n");
6729 : return false;
6730 : }
6731 0 : is_invariant &= (dt[2] == vect_external_def
6732 0 : || dt[2] == vect_constant_def);
6733 0 : if (vectype3
6734 0 : && (maybe_ne (nunits_out, TYPE_VECTOR_SUBPARTS (vectype3))
6735 0 : || !tree_nop_conversion_p (TREE_TYPE (vectype_out),
6736 0 : TREE_TYPE (vectype3))))
6737 : return false;
6738 : }
6739 :
6740 : /* Multiple types in SLP are handled by creating the appropriate number of
6741 : vectorized stmts for each SLP node. */
6742 641650 : auto vec_num = vect_get_num_copies (vinfo, slp_node);
6743 :
6744 : /* Reject attempts to combine mask types with nonmask types, e.g. if
6745 : we have an AND between a (nonmask) boolean loaded from memory and
6746 : a (mask) boolean result of a comparison.
6747 :
6748 : TODO: We could easily fix these cases up using pattern statements. */
6749 641650 : if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
6750 981089 : || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
6751 1283300 : || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
6752 : {
6753 0 : if (dump_enabled_p ())
6754 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6755 : "mixed mask and nonmask vector types\n");
6756 : return false;
6757 : }
6758 :
6759 : /* Supportable by target? */
6760 :
6761 641650 : vec_mode = TYPE_MODE (vectype);
6762 641650 : optab = optab_for_tree_code (code, vectype, optab_default);
6763 641650 : if (!optab)
6764 : {
6765 69736 : if (dump_enabled_p ())
6766 5874 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6767 : "no optab for %s and %T.\n",
6768 : get_tree_code_name (code), vectype);
6769 : return false;
6770 : }
6771 571914 : target_support_p = can_implement_p (optab, vec_mode);
6772 :
6773 571914 : bool using_emulated_vectors_p = vect_emulated_vector_p (vectype);
6774 571914 : if (!target_support_p || using_emulated_vectors_p)
6775 : {
6776 30685 : if (dump_enabled_p ())
6777 1152 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6778 : "op not supported by target.\n");
6779 : /* When vec_mode is not a vector mode and we verified ops we
6780 : do not have to lower like AND are natively supported let
6781 : those through even when the mode isn't word_mode. For
6782 : ops we have to lower the lowering code assumes we are
6783 : dealing with word_mode. */
6784 61370 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype))
6785 30543 : || !GET_MODE_SIZE (vec_mode).is_constant ()
6786 30543 : || (((code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
6787 25352 : || !target_support_p)
6788 65621 : && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
6789 : /* Check only during analysis. */
6790 42828 : || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
6791 : {
6792 30081 : if (dump_enabled_p ())
6793 1150 : dump_printf (MSG_NOTE, "using word mode not possible.\n");
6794 : return false;
6795 : }
6796 604 : if (dump_enabled_p ())
6797 2 : dump_printf_loc (MSG_NOTE, vect_location,
6798 : "proceeding using word mode.\n");
6799 : using_emulated_vectors_p = true;
6800 : }
6801 :
6802 541833 : int reduc_idx = SLP_TREE_REDUC_IDX (slp_node);
6803 541833 : vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
6804 437251 : vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
6805 541833 : internal_fn cond_fn = get_conditional_internal_fn (code);
6806 541833 : internal_fn cond_len_fn = get_conditional_len_internal_fn (code);
6807 :
6808 : /* If operating on inactive elements could generate spurious traps,
6809 : we need to restrict the operation to active lanes. Note that this
6810 : specifically doesn't apply to unhoisted invariants, since they
6811 : operate on the same value for every lane.
6812 :
6813 : Similarly, if this operation is part of a reduction, a fully-masked
6814 : loop should only change the active lanes of the reduction chain,
6815 : keeping the inactive lanes as-is. */
6816 512916 : bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
6817 985451 : || reduc_idx >= 0);
6818 :
6819 541833 : if (cost_vec) /* transformation not required. */
6820 : {
6821 424175 : if (loop_vinfo
6822 333456 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
6823 90424 : && mask_out_inactive)
6824 : {
6825 20398 : if (cond_len_fn != IFN_LAST
6826 20398 : && direct_internal_fn_supported_p (cond_len_fn, vectype,
6827 : OPTIMIZE_FOR_SPEED))
6828 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype,
6829 : 1);
6830 20398 : else if (cond_fn != IFN_LAST
6831 20398 : && direct_internal_fn_supported_p (cond_fn, vectype,
6832 : OPTIMIZE_FOR_SPEED))
6833 8505 : vect_record_loop_mask (loop_vinfo, masks, vec_num,
6834 : vectype, NULL);
6835 : else
6836 : {
6837 11893 : if (dump_enabled_p ())
6838 607 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6839 : "can't use a fully-masked loop because no"
6840 : " conditional operation is available.\n");
6841 11893 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
6842 : }
6843 : }
6844 :
6845 : /* Put types on constant and invariant SLP children. */
6846 424175 : if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
6847 424095 : || !vect_maybe_update_slp_op_vectype (slp_op1, vectype)
6848 848176 : || !vect_maybe_update_slp_op_vectype (slp_op2, vectype))
6849 : {
6850 174 : if (dump_enabled_p ())
6851 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6852 : "incompatible vector types for invariants\n");
6853 : return false;
6854 : }
6855 :
6856 424001 : SLP_TREE_TYPE (slp_node) = op_vec_info_type;
6857 424001 : DUMP_VECT_SCOPE ("vectorizable_operation");
6858 424001 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
6859 424001 : if (using_emulated_vectors_p)
6860 : {
6861 : /* The above vect_model_simple_cost call handles constants
6862 : in the prologue and (mis-)costs one of the stmts as
6863 : vector stmt. See below for the actual lowering that will
6864 : be applied. */
6865 602 : unsigned n = vect_get_num_copies (vinfo, slp_node);
6866 602 : switch (code)
6867 : {
6868 213 : case PLUS_EXPR:
6869 213 : n *= 5;
6870 213 : break;
6871 352 : case MINUS_EXPR:
6872 352 : n *= 6;
6873 352 : break;
6874 0 : case NEGATE_EXPR:
6875 0 : n *= 4;
6876 0 : break;
6877 : default:
6878 : /* Bit operations do not have extra cost and are accounted
6879 : as vector stmt by vect_model_simple_cost. */
6880 : n = 0;
6881 : break;
6882 : }
6883 565 : if (n != 0)
6884 : {
6885 : /* We also need to materialize two large constants. */
6886 565 : record_stmt_cost (cost_vec, 2, scalar_stmt, stmt_info,
6887 : 0, vect_prologue);
6888 565 : record_stmt_cost (cost_vec, n, scalar_stmt, stmt_info,
6889 : 0, vect_body);
6890 : }
6891 : }
6892 424001 : return true;
6893 : }
6894 :
6895 : /* Transform. */
6896 :
6897 117658 : if (dump_enabled_p ())
6898 16935 : dump_printf_loc (MSG_NOTE, vect_location,
6899 : "transform binary/unary operation.\n");
6900 :
6901 117658 : bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
6902 103795 : bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
6903 :
6904 : /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6905 : vectors with unsigned elements, but the result is signed. So, we
6906 : need to compute the MINUS_EXPR into vectype temporary and
6907 : VIEW_CONVERT_EXPR it into the final vectype_out result. */
6908 117658 : tree vec_cvt_dest = NULL_TREE;
6909 117658 : if (orig_code == POINTER_DIFF_EXPR)
6910 : {
6911 123 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6912 123 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6913 : }
6914 : /* For reduction operations with undefined overflow behavior make sure to
6915 : pun them to unsigned since we change the order of evaluation.
6916 : ??? Avoid for in-order reductions? */
6917 117535 : else if (arith_code_with_undefined_signed_overflow (orig_code)
6918 100050 : && ANY_INTEGRAL_TYPE_P (vectype)
6919 48659 : && TYPE_OVERFLOW_UNDEFINED (vectype)
6920 143679 : && SLP_TREE_REDUC_IDX (slp_node) != -1)
6921 : {
6922 2482 : gcc_assert (orig_code == PLUS_EXPR || orig_code == MINUS_EXPR
6923 : || orig_code == MULT_EXPR || orig_code == POINTER_PLUS_EXPR);
6924 2482 : vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
6925 2482 : vectype = unsigned_type_for (vectype);
6926 2482 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6927 : }
6928 : /* Handle def. */
6929 : else
6930 115053 : vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
6931 :
6932 117658 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0,
6933 : slp_op1, &vec_oprnds1, slp_op2, &vec_oprnds2);
6934 : /* Arguments are ready. Create the new vector stmt. */
6935 377130 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
6936 : {
6937 141814 : gimple *new_stmt = NULL;
6938 283628 : vop1 = ((op_type == binary_op || op_type == ternary_op)
6939 141814 : ? vec_oprnds1[i] : NULL_TREE);
6940 141814 : vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE);
6941 :
6942 141814 : if (vec_cvt_dest
6943 141814 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop0)))
6944 : {
6945 2935 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop0);
6946 2935 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6947 : new_temp);
6948 2935 : new_temp = make_ssa_name (vec_dest, new_stmt);
6949 2935 : gimple_assign_set_lhs (new_stmt, new_temp);
6950 2935 : vect_finish_stmt_generation (vinfo, stmt_info,
6951 : new_stmt, gsi);
6952 2935 : vop0 = new_temp;
6953 : }
6954 141814 : if (vop1
6955 139030 : && vec_cvt_dest
6956 144887 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop1)))
6957 : {
6958 2935 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop1);
6959 2935 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6960 : new_temp);
6961 2935 : new_temp = make_ssa_name (vec_dest, new_stmt);
6962 2935 : gimple_assign_set_lhs (new_stmt, new_temp);
6963 2935 : vect_finish_stmt_generation (vinfo, stmt_info,
6964 : new_stmt, gsi);
6965 2935 : vop1 = new_temp;
6966 : }
6967 141814 : if (vop2
6968 0 : && vec_cvt_dest
6969 141814 : && !useless_type_conversion_p (vectype, TREE_TYPE (vop2)))
6970 : {
6971 0 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype, vop2);
6972 0 : new_stmt = gimple_build_assign (vec_dest, VIEW_CONVERT_EXPR,
6973 : new_temp);
6974 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6975 0 : gimple_assign_set_lhs (new_stmt, new_temp);
6976 0 : vect_finish_stmt_generation (vinfo, stmt_info,
6977 : new_stmt, gsi);
6978 0 : vop2 = new_temp;
6979 : }
6980 :
6981 141814 : if (using_emulated_vectors_p)
6982 : {
6983 : /* Lower the operation. This follows vector lowering. */
6984 2 : tree word_type = build_nonstandard_integer_type
6985 2 : (GET_MODE_BITSIZE (vec_mode).to_constant (), 1);
6986 2 : tree wvop0 = make_ssa_name (word_type);
6987 2 : new_stmt = gimple_build_assign (wvop0, VIEW_CONVERT_EXPR,
6988 : build1 (VIEW_CONVERT_EXPR,
6989 : word_type, vop0));
6990 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6991 2 : tree wvop1 = NULL_TREE;
6992 2 : if (vop1)
6993 : {
6994 2 : wvop1 = make_ssa_name (word_type);
6995 2 : new_stmt = gimple_build_assign (wvop1, VIEW_CONVERT_EXPR,
6996 : build1 (VIEW_CONVERT_EXPR,
6997 : word_type, vop1));
6998 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
6999 : }
7000 :
7001 2 : tree result_low;
7002 2 : if (code == PLUS_EXPR || code == MINUS_EXPR || code == NEGATE_EXPR)
7003 : {
7004 1 : unsigned int width = vector_element_bits (vectype);
7005 1 : tree inner_type = TREE_TYPE (vectype);
7006 1 : HOST_WIDE_INT max = GET_MODE_MASK (TYPE_MODE (inner_type));
7007 1 : tree low_bits
7008 1 : = build_replicated_int_cst (word_type, width, max >> 1);
7009 1 : tree high_bits
7010 2 : = build_replicated_int_cst (word_type,
7011 1 : width, max & ~(max >> 1));
7012 1 : tree signs;
7013 1 : if (code == PLUS_EXPR || code == MINUS_EXPR)
7014 : {
7015 1 : signs = make_ssa_name (word_type);
7016 1 : new_stmt = gimple_build_assign (signs,
7017 : BIT_XOR_EXPR, wvop0, wvop1);
7018 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7019 1 : tree b_low = make_ssa_name (word_type);
7020 1 : new_stmt = gimple_build_assign (b_low, BIT_AND_EXPR,
7021 : wvop1, low_bits);
7022 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7023 1 : tree a_low = make_ssa_name (word_type);
7024 1 : if (code == PLUS_EXPR)
7025 1 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7026 : wvop0, low_bits);
7027 : else
7028 0 : new_stmt = gimple_build_assign (a_low, BIT_IOR_EXPR,
7029 : wvop0, high_bits);
7030 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7031 1 : if (code == MINUS_EXPR)
7032 : {
7033 0 : new_stmt = gimple_build_assign (NULL_TREE,
7034 : BIT_NOT_EXPR, signs);
7035 0 : signs = make_ssa_name (word_type);
7036 0 : gimple_assign_set_lhs (new_stmt, signs);
7037 0 : vect_finish_stmt_generation (vinfo, stmt_info,
7038 : new_stmt, gsi);
7039 : }
7040 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7041 : signs, high_bits);
7042 1 : signs = make_ssa_name (word_type);
7043 1 : gimple_assign_set_lhs (new_stmt, signs);
7044 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7045 1 : result_low = make_ssa_name (word_type);
7046 1 : new_stmt = gimple_build_assign (result_low, code,
7047 : a_low, b_low);
7048 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7049 : }
7050 : else /* if (code == NEGATE_EXPR) */
7051 : {
7052 0 : tree a_low = make_ssa_name (word_type);
7053 0 : new_stmt = gimple_build_assign (a_low, BIT_AND_EXPR,
7054 : wvop0, low_bits);
7055 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7056 0 : signs = make_ssa_name (word_type);
7057 0 : new_stmt = gimple_build_assign (signs, BIT_NOT_EXPR, wvop0);
7058 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7059 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR,
7060 : signs, high_bits);
7061 0 : signs = make_ssa_name (word_type);
7062 0 : gimple_assign_set_lhs (new_stmt, signs);
7063 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7064 0 : result_low = make_ssa_name (word_type);
7065 0 : new_stmt = gimple_build_assign (result_low,
7066 : MINUS_EXPR, high_bits, a_low);
7067 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7068 : }
7069 1 : new_stmt = gimple_build_assign (NULL_TREE, BIT_XOR_EXPR,
7070 : result_low, signs);
7071 1 : result_low = make_ssa_name (word_type);
7072 1 : gimple_assign_set_lhs (new_stmt, result_low);
7073 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7074 : }
7075 : else
7076 : {
7077 1 : new_stmt = gimple_build_assign (NULL_TREE, code, wvop0, wvop1);
7078 1 : result_low = make_ssa_name (word_type);
7079 1 : gimple_assign_set_lhs (new_stmt, result_low);
7080 1 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7081 :
7082 : }
7083 2 : new_stmt = gimple_build_assign (NULL_TREE, VIEW_CONVERT_EXPR,
7084 : build1 (VIEW_CONVERT_EXPR,
7085 : vectype, result_low));
7086 2 : new_temp = make_ssa_name (vectype);
7087 2 : gimple_assign_set_lhs (new_stmt, new_temp);
7088 2 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7089 : }
7090 141812 : else if ((masked_loop_p || len_loop_p) && mask_out_inactive)
7091 : {
7092 16 : tree mask;
7093 16 : if (masked_loop_p)
7094 16 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7095 : vec_num, vectype, i);
7096 : else
7097 : /* Dummy mask. */
7098 0 : mask = build_minus_one_cst (truth_type_for (vectype));
7099 16 : auto_vec<tree> vops (6);
7100 16 : vops.quick_push (mask);
7101 16 : vops.quick_push (vop0);
7102 16 : if (vop1)
7103 16 : vops.quick_push (vop1);
7104 16 : if (vop2)
7105 0 : vops.quick_push (vop2);
7106 16 : if (reduc_idx >= 0)
7107 : {
7108 : /* Perform the operation on active elements only and take
7109 : inactive elements from the reduction chain input. */
7110 8 : gcc_assert (!vop2);
7111 8 : vops.quick_push (reduc_idx == 1 ? vop1 : vop0);
7112 : }
7113 : else
7114 : {
7115 8 : auto else_value = targetm.preferred_else_value
7116 8 : (cond_fn, vectype, vops.length () - 1, &vops[1]);
7117 8 : vops.quick_push (else_value);
7118 : }
7119 16 : if (len_loop_p)
7120 : {
7121 0 : tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
7122 0 : vec_num, vectype, i, 1, true);
7123 0 : signed char biasval
7124 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
7125 0 : tree bias = build_int_cst (intQI_type_node, biasval);
7126 0 : vops.quick_push (len);
7127 0 : vops.quick_push (bias);
7128 : }
7129 16 : gcall *call
7130 16 : = gimple_build_call_internal_vec (masked_loop_p ? cond_fn
7131 : : cond_len_fn,
7132 : vops);
7133 16 : new_temp = make_ssa_name (vec_dest, call);
7134 16 : gimple_call_set_lhs (call, new_temp);
7135 16 : gimple_call_set_nothrow (call, true);
7136 16 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
7137 16 : new_stmt = call;
7138 16 : }
7139 : else
7140 : {
7141 141796 : tree mask = NULL_TREE;
7142 : /* When combining two masks check if either of them is elsewhere
7143 : combined with a loop mask, if that's the case we can mark that the
7144 : new combined mask doesn't need to be combined with a loop mask. */
7145 141796 : if (masked_loop_p
7146 141796 : && code == BIT_AND_EXPR
7147 141796 : && VECTOR_BOOLEAN_TYPE_P (vectype))
7148 : {
7149 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
7150 : {
7151 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7152 : vec_num, vectype, i);
7153 :
7154 0 : vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7155 : vop0, gsi);
7156 : }
7157 :
7158 8 : if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
7159 : {
7160 0 : mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
7161 : vec_num, vectype, i);
7162 :
7163 0 : vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
7164 : vop1, gsi);
7165 : }
7166 : }
7167 :
7168 141796 : new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
7169 141796 : new_temp = make_ssa_name (vec_dest, new_stmt);
7170 141796 : gimple_assign_set_lhs (new_stmt, new_temp);
7171 141796 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
7172 141796 : if (using_emulated_vectors_p)
7173 : suppress_warning (new_stmt, OPT_Wvector_operation_performance);
7174 :
7175 : /* Enter the combined value into the vector cond hash so we don't
7176 : AND it with a loop mask again. */
7177 141796 : if (mask)
7178 0 : loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
7179 : }
7180 :
7181 141814 : if (vec_cvt_dest)
7182 : {
7183 3073 : new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
7184 3073 : new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
7185 : new_temp);
7186 3073 : new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
7187 3073 : gimple_assign_set_lhs (new_stmt, new_temp);
7188 3073 : vect_finish_stmt_generation (vinfo, stmt_info,
7189 : new_stmt, gsi);
7190 : }
7191 :
7192 141814 : slp_node->push_vec_def (new_stmt);
7193 : }
7194 :
7195 117658 : vec_oprnds0.release ();
7196 117658 : vec_oprnds1.release ();
7197 117658 : vec_oprnds2.release ();
7198 :
7199 117658 : return true;
7200 : }
7201 :
7202 : /* A helper function to ensure data reference DR_INFO's base alignment. */
7203 :
7204 : static void
7205 729069 : ensure_base_align (dr_vec_info *dr_info)
7206 : {
7207 : /* Alignment is only analyzed for the first element of a DR group,
7208 : use that to look at base alignment we need to enforce. */
7209 729069 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
7210 608041 : dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
7211 :
7212 729069 : gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
7213 :
7214 729069 : if (dr_info->base_misaligned)
7215 : {
7216 117977 : tree base_decl = dr_info->base_decl;
7217 :
7218 : // We should only be able to increase the alignment of a base object if
7219 : // we know what its new alignment should be at compile time.
7220 117977 : unsigned HOST_WIDE_INT align_base_to =
7221 117977 : DR_TARGET_ALIGNMENT (dr_info).to_constant () * BITS_PER_UNIT;
7222 :
7223 117977 : if (decl_in_symtab_p (base_decl))
7224 3281 : symtab_node::get (base_decl)->increase_alignment (align_base_to);
7225 114696 : else if (DECL_ALIGN (base_decl) < align_base_to)
7226 : {
7227 95211 : SET_DECL_ALIGN (base_decl, align_base_to);
7228 95211 : DECL_USER_ALIGN (base_decl) = 1;
7229 : }
7230 117977 : dr_info->base_misaligned = false;
7231 : }
7232 729069 : }
7233 :
7234 :
7235 : /* Function get_group_alias_ptr_type.
7236 :
7237 : Return the alias type for the group starting at FIRST_STMT_INFO. */
7238 :
7239 : static tree
7240 1676664 : get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
7241 : {
7242 1676664 : struct data_reference *first_dr, *next_dr;
7243 :
7244 1676664 : first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
7245 1676664 : stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
7246 3980125 : while (next_stmt_info)
7247 : {
7248 2494173 : next_dr = STMT_VINFO_DATA_REF (next_stmt_info);
7249 4988346 : if (get_alias_set (DR_REF (first_dr))
7250 2494173 : != get_alias_set (DR_REF (next_dr)))
7251 : {
7252 190712 : if (dump_enabled_p ())
7253 30 : dump_printf_loc (MSG_NOTE, vect_location,
7254 : "conflicting alias set types.\n");
7255 190712 : return ptr_type_node;
7256 : }
7257 2303461 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7258 : }
7259 1485952 : return reference_alias_ptr_type (DR_REF (first_dr));
7260 : }
7261 :
7262 :
7263 : /* Function scan_operand_equal_p.
7264 :
7265 : Helper function for check_scan_store. Compare two references
7266 : with .GOMP_SIMD_LANE bases. */
7267 :
7268 : static bool
7269 1284 : scan_operand_equal_p (tree ref1, tree ref2)
7270 : {
7271 1284 : tree ref[2] = { ref1, ref2 };
7272 1284 : poly_int64 bitsize[2], bitpos[2];
7273 : tree offset[2], base[2];
7274 3852 : for (int i = 0; i < 2; ++i)
7275 : {
7276 2568 : machine_mode mode;
7277 2568 : int unsignedp, reversep, volatilep = 0;
7278 2568 : base[i] = get_inner_reference (ref[i], &bitsize[i], &bitpos[i],
7279 : &offset[i], &mode, &unsignedp,
7280 : &reversep, &volatilep);
7281 2568 : if (reversep || volatilep || maybe_ne (bitpos[i], 0))
7282 0 : return false;
7283 2568 : if (TREE_CODE (base[i]) == MEM_REF
7284 42 : && offset[i] == NULL_TREE
7285 2610 : && TREE_CODE (TREE_OPERAND (base[i], 0)) == SSA_NAME)
7286 : {
7287 42 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base[i], 0));
7288 42 : if (is_gimple_assign (def_stmt)
7289 42 : && gimple_assign_rhs_code (def_stmt) == POINTER_PLUS_EXPR
7290 42 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == ADDR_EXPR
7291 84 : && TREE_CODE (gimple_assign_rhs2 (def_stmt)) == SSA_NAME)
7292 : {
7293 42 : if (maybe_ne (mem_ref_offset (base[i]), 0))
7294 : return false;
7295 42 : base[i] = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
7296 42 : offset[i] = gimple_assign_rhs2 (def_stmt);
7297 : }
7298 : }
7299 : }
7300 :
7301 1284 : if (!operand_equal_p (base[0], base[1], 0))
7302 : return false;
7303 934 : if (maybe_ne (bitsize[0], bitsize[1]))
7304 : return false;
7305 934 : if (offset[0] != offset[1])
7306 : {
7307 916 : if (!offset[0] || !offset[1])
7308 : return false;
7309 916 : if (!operand_equal_p (offset[0], offset[1], 0))
7310 : {
7311 : tree step[2];
7312 0 : for (int i = 0; i < 2; ++i)
7313 : {
7314 0 : step[i] = integer_one_node;
7315 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7316 : {
7317 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7318 0 : if (is_gimple_assign (def_stmt)
7319 0 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
7320 0 : && (TREE_CODE (gimple_assign_rhs2 (def_stmt))
7321 : == INTEGER_CST))
7322 : {
7323 0 : step[i] = gimple_assign_rhs2 (def_stmt);
7324 0 : offset[i] = gimple_assign_rhs1 (def_stmt);
7325 : }
7326 : }
7327 0 : else if (TREE_CODE (offset[i]) == MULT_EXPR)
7328 : {
7329 0 : step[i] = TREE_OPERAND (offset[i], 1);
7330 0 : offset[i] = TREE_OPERAND (offset[i], 0);
7331 : }
7332 0 : tree rhs1 = NULL_TREE;
7333 0 : if (TREE_CODE (offset[i]) == SSA_NAME)
7334 : {
7335 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (offset[i]);
7336 0 : if (gimple_assign_cast_p (def_stmt))
7337 0 : rhs1 = gimple_assign_rhs1 (def_stmt);
7338 : }
7339 0 : else if (CONVERT_EXPR_P (offset[i]))
7340 0 : rhs1 = TREE_OPERAND (offset[i], 0);
7341 0 : if (rhs1
7342 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7343 0 : && INTEGRAL_TYPE_P (TREE_TYPE (offset[i]))
7344 0 : && (TYPE_PRECISION (TREE_TYPE (offset[i]))
7345 0 : >= TYPE_PRECISION (TREE_TYPE (rhs1))))
7346 0 : offset[i] = rhs1;
7347 : }
7348 0 : if (!operand_equal_p (offset[0], offset[1], 0)
7349 0 : || !operand_equal_p (step[0], step[1], 0))
7350 0 : return false;
7351 : }
7352 : }
7353 : return true;
7354 : }
7355 :
7356 :
7357 : enum scan_store_kind {
7358 : /* Normal permutation. */
7359 : scan_store_kind_perm,
7360 :
7361 : /* Whole vector left shift permutation with zero init. */
7362 : scan_store_kind_lshift_zero,
7363 :
7364 : /* Whole vector left shift permutation and VEC_COND_EXPR. */
7365 : scan_store_kind_lshift_cond
7366 : };
7367 :
7368 : /* Function check_scan_store.
7369 :
7370 : Verify if we can perform the needed permutations or whole vector shifts.
7371 : Return -1 on failure, otherwise exact log2 of vectype's nunits.
7372 : USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7373 : to do at each step. */
7374 :
7375 : static int
7376 1024 : scan_store_can_perm_p (tree vectype, tree init,
7377 : vec<enum scan_store_kind> *use_whole_vector = NULL)
7378 : {
7379 1024 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7380 1024 : unsigned HOST_WIDE_INT nunits;
7381 1024 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7382 : return -1;
7383 1024 : int units_log2 = exact_log2 (nunits);
7384 1024 : if (units_log2 <= 0)
7385 : return -1;
7386 :
7387 : int i;
7388 : enum scan_store_kind whole_vector_shift_kind = scan_store_kind_perm;
7389 4784 : for (i = 0; i <= units_log2; ++i)
7390 : {
7391 3760 : unsigned HOST_WIDE_INT j, k;
7392 3760 : enum scan_store_kind kind = scan_store_kind_perm;
7393 3760 : vec_perm_builder sel (nunits, nunits, 1);
7394 3760 : sel.quick_grow (nunits);
7395 3760 : if (i == units_log2)
7396 : {
7397 9728 : for (j = 0; j < nunits; ++j)
7398 8704 : sel[j] = nunits - 1;
7399 : }
7400 : else
7401 : {
7402 10416 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7403 7680 : sel[j] = j;
7404 26416 : for (k = 0; j < nunits; ++j, ++k)
7405 23680 : sel[j] = nunits + k;
7406 : }
7407 6496 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7408 3760 : if (!can_vec_perm_const_p (vec_mode, vec_mode, indices))
7409 : {
7410 0 : if (i == units_log2)
7411 : return -1;
7412 :
7413 0 : if (whole_vector_shift_kind == scan_store_kind_perm)
7414 : {
7415 0 : if (!can_implement_p (vec_shl_optab, vec_mode))
7416 : return -1;
7417 0 : whole_vector_shift_kind = scan_store_kind_lshift_zero;
7418 : /* Whole vector shifts shift in zeros, so if init is all zero
7419 : constant, there is no need to do anything further. */
7420 0 : if ((TREE_CODE (init) != INTEGER_CST
7421 0 : && TREE_CODE (init) != REAL_CST)
7422 0 : || !initializer_zerop (init))
7423 : {
7424 0 : tree masktype = truth_type_for (vectype);
7425 0 : if (!expand_vec_cond_expr_p (vectype, masktype))
7426 : return -1;
7427 : whole_vector_shift_kind = scan_store_kind_lshift_cond;
7428 : }
7429 : }
7430 0 : kind = whole_vector_shift_kind;
7431 : }
7432 3760 : if (use_whole_vector)
7433 : {
7434 1880 : if (kind != scan_store_kind_perm && use_whole_vector->is_empty ())
7435 0 : use_whole_vector->safe_grow_cleared (i, true);
7436 5640 : if (kind != scan_store_kind_perm || !use_whole_vector->is_empty ())
7437 0 : use_whole_vector->safe_push (kind);
7438 : }
7439 3760 : }
7440 :
7441 : return units_log2;
7442 : }
7443 :
7444 :
7445 : /* Function check_scan_store.
7446 :
7447 : Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7448 :
7449 : static bool
7450 1076 : check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
7451 : enum vect_def_type rhs_dt, slp_tree slp_node,
7452 : slp_tree mask_node,
7453 : vect_memory_access_type memory_access_type)
7454 : {
7455 1076 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7456 1076 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7457 1076 : tree ref_type;
7458 :
7459 1076 : gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
7460 1076 : if (SLP_TREE_LANES (slp_node) > 1
7461 1076 : || mask_node
7462 1076 : || memory_access_type != VMAT_CONTIGUOUS
7463 1076 : || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
7464 1076 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
7465 1076 : || loop_vinfo == NULL
7466 1076 : || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
7467 1076 : || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
7468 1076 : || STMT_VINFO_GROUPED_ACCESS (stmt_info)
7469 1076 : || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
7470 1076 : || !integer_zerop (DR_INIT (dr_info->dr))
7471 1076 : || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
7472 2152 : || !alias_sets_conflict_p (get_alias_set (vectype),
7473 1076 : get_alias_set (TREE_TYPE (ref_type))))
7474 : {
7475 0 : if (dump_enabled_p ())
7476 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7477 : "unsupported OpenMP scan store.\n");
7478 : return false;
7479 : }
7480 :
7481 : /* We need to pattern match code built by OpenMP lowering and simplified
7482 : by following optimizations into something we can handle.
7483 : #pragma omp simd reduction(inscan,+:r)
7484 : for (...)
7485 : {
7486 : r += something ();
7487 : #pragma omp scan inclusive (r)
7488 : use (r);
7489 : }
7490 : shall have body with:
7491 : // Initialization for input phase, store the reduction initializer:
7492 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7493 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7494 : D.2042[_21] = 0;
7495 : // Actual input phase:
7496 : ...
7497 : r.0_5 = D.2042[_20];
7498 : _6 = _4 + r.0_5;
7499 : D.2042[_20] = _6;
7500 : // Initialization for scan phase:
7501 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7502 : _26 = D.2043[_25];
7503 : _27 = D.2042[_25];
7504 : _28 = _26 + _27;
7505 : D.2043[_25] = _28;
7506 : D.2042[_25] = _28;
7507 : // Actual scan phase:
7508 : ...
7509 : r.1_8 = D.2042[_20];
7510 : ...
7511 : The "omp simd array" variable D.2042 holds the privatized copy used
7512 : inside of the loop and D.2043 is another one that holds copies of
7513 : the current original list item. The separate GOMP_SIMD_LANE ifn
7514 : kinds are there in order to allow optimizing the initializer store
7515 : and combiner sequence, e.g. if it is originally some C++ish user
7516 : defined reduction, but allow the vectorizer to pattern recognize it
7517 : and turn into the appropriate vectorized scan.
7518 :
7519 : For exclusive scan, this is slightly different:
7520 : #pragma omp simd reduction(inscan,+:r)
7521 : for (...)
7522 : {
7523 : use (r);
7524 : #pragma omp scan exclusive (r)
7525 : r += something ();
7526 : }
7527 : shall have body with:
7528 : // Initialization for input phase, store the reduction initializer:
7529 : _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7530 : _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7531 : D.2042[_21] = 0;
7532 : // Actual input phase:
7533 : ...
7534 : r.0_5 = D.2042[_20];
7535 : _6 = _4 + r.0_5;
7536 : D.2042[_20] = _6;
7537 : // Initialization for scan phase:
7538 : _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7539 : _26 = D.2043[_25];
7540 : D.2044[_25] = _26;
7541 : _27 = D.2042[_25];
7542 : _28 = _26 + _27;
7543 : D.2043[_25] = _28;
7544 : // Actual scan phase:
7545 : ...
7546 : r.1_8 = D.2044[_20];
7547 : ... */
7548 :
7549 1076 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 2)
7550 : {
7551 : /* Match the D.2042[_21] = 0; store above. Just require that
7552 : it is a constant or external definition store. */
7553 564 : if (rhs_dt != vect_constant_def && rhs_dt != vect_external_def)
7554 : {
7555 0 : fail_init:
7556 0 : if (dump_enabled_p ())
7557 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7558 : "unsupported OpenMP scan initializer store.\n");
7559 : return false;
7560 : }
7561 :
7562 564 : if (! loop_vinfo->scan_map)
7563 322 : loop_vinfo->scan_map = new hash_map<tree, tree>;
7564 564 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7565 564 : tree &cached = loop_vinfo->scan_map->get_or_insert (var);
7566 564 : if (cached)
7567 0 : goto fail_init;
7568 564 : cached = gimple_assign_rhs1 (STMT_VINFO_STMT (stmt_info));
7569 :
7570 : /* These stores can be vectorized normally. */
7571 564 : return true;
7572 : }
7573 :
7574 512 : if (rhs_dt != vect_internal_def)
7575 : {
7576 0 : fail:
7577 0 : if (dump_enabled_p ())
7578 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7579 : "unsupported OpenMP scan combiner pattern.\n");
7580 : return false;
7581 : }
7582 :
7583 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7584 512 : tree rhs = gimple_assign_rhs1 (stmt);
7585 512 : if (TREE_CODE (rhs) != SSA_NAME)
7586 0 : goto fail;
7587 :
7588 512 : gimple *other_store_stmt = NULL;
7589 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7590 512 : bool inscan_var_store
7591 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7592 :
7593 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7594 : {
7595 252 : if (!inscan_var_store)
7596 : {
7597 126 : use_operand_p use_p;
7598 126 : imm_use_iterator iter;
7599 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7600 : {
7601 252 : gimple *use_stmt = USE_STMT (use_p);
7602 252 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7603 126 : continue;
7604 126 : if (gimple_bb (use_stmt) != gimple_bb (stmt)
7605 126 : || !is_gimple_assign (use_stmt)
7606 126 : || gimple_assign_rhs_class (use_stmt) != GIMPLE_BINARY_RHS
7607 126 : || other_store_stmt
7608 252 : || TREE_CODE (gimple_assign_lhs (use_stmt)) != SSA_NAME)
7609 0 : goto fail;
7610 126 : other_store_stmt = use_stmt;
7611 0 : }
7612 126 : if (other_store_stmt == NULL)
7613 0 : goto fail;
7614 126 : rhs = gimple_assign_lhs (other_store_stmt);
7615 126 : if (!single_imm_use (rhs, &use_p, &other_store_stmt))
7616 0 : goto fail;
7617 : }
7618 : }
7619 260 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3)
7620 : {
7621 260 : use_operand_p use_p;
7622 260 : imm_use_iterator iter;
7623 780 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7624 : {
7625 520 : gimple *use_stmt = USE_STMT (use_p);
7626 520 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7627 260 : continue;
7628 260 : if (other_store_stmt)
7629 0 : goto fail;
7630 260 : other_store_stmt = use_stmt;
7631 260 : }
7632 : }
7633 : else
7634 0 : goto fail;
7635 :
7636 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7637 512 : if (gimple_bb (def_stmt) != gimple_bb (stmt)
7638 512 : || !is_gimple_assign (def_stmt)
7639 1024 : || gimple_assign_rhs_class (def_stmt) != GIMPLE_BINARY_RHS)
7640 0 : goto fail;
7641 :
7642 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7643 : /* For pointer addition, we should use the normal plus for the vector
7644 : operation. */
7645 512 : switch (code)
7646 : {
7647 0 : case POINTER_PLUS_EXPR:
7648 0 : code = PLUS_EXPR;
7649 0 : break;
7650 0 : case MULT_HIGHPART_EXPR:
7651 0 : goto fail;
7652 : default:
7653 : break;
7654 : }
7655 512 : if (TREE_CODE_LENGTH (code) != binary_op || !commutative_tree_code (code))
7656 0 : goto fail;
7657 :
7658 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7659 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7660 512 : if (TREE_CODE (rhs1) != SSA_NAME || TREE_CODE (rhs2) != SSA_NAME)
7661 0 : goto fail;
7662 :
7663 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7664 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7665 512 : if (gimple_bb (load1_stmt) != gimple_bb (stmt)
7666 512 : || !gimple_assign_load_p (load1_stmt)
7667 512 : || gimple_bb (load2_stmt) != gimple_bb (stmt)
7668 1024 : || !gimple_assign_load_p (load2_stmt))
7669 0 : goto fail;
7670 :
7671 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7672 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7673 512 : if (load1_stmt_info == NULL
7674 512 : || load2_stmt_info == NULL
7675 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load1_stmt_info)
7676 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info))
7677 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (load2_stmt_info)
7678 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7679 0 : goto fail;
7680 :
7681 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && inscan_var_store)
7682 : {
7683 126 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7684 126 : if (TREE_CODE (DR_BASE_ADDRESS (load1_dr_info->dr)) != ADDR_EXPR
7685 126 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0)))
7686 0 : goto fail;
7687 126 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7688 126 : tree lrhs;
7689 126 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7690 : lrhs = rhs1;
7691 : else
7692 16 : lrhs = rhs2;
7693 126 : use_operand_p use_p;
7694 126 : imm_use_iterator iter;
7695 378 : FOR_EACH_IMM_USE_FAST (use_p, iter, lrhs)
7696 : {
7697 252 : gimple *use_stmt = USE_STMT (use_p);
7698 252 : if (use_stmt == def_stmt || is_gimple_debug (use_stmt))
7699 126 : continue;
7700 126 : if (other_store_stmt)
7701 0 : goto fail;
7702 126 : other_store_stmt = use_stmt;
7703 126 : }
7704 : }
7705 :
7706 512 : if (other_store_stmt == NULL)
7707 0 : goto fail;
7708 512 : if (gimple_bb (other_store_stmt) != gimple_bb (stmt)
7709 512 : || !gimple_store_p (other_store_stmt))
7710 0 : goto fail;
7711 :
7712 512 : stmt_vec_info other_store_stmt_info
7713 512 : = loop_vinfo->lookup_stmt (other_store_stmt);
7714 512 : if (other_store_stmt_info == NULL
7715 512 : || (STMT_VINFO_SIMD_LANE_ACCESS_P (other_store_stmt_info)
7716 512 : != STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)))
7717 0 : goto fail;
7718 :
7719 512 : gimple *stmt1 = stmt;
7720 512 : gimple *stmt2 = other_store_stmt;
7721 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7722 : std::swap (stmt1, stmt2);
7723 512 : if (scan_operand_equal_p (gimple_assign_lhs (stmt1),
7724 : gimple_assign_rhs1 (load2_stmt)))
7725 : {
7726 162 : std::swap (rhs1, rhs2);
7727 162 : std::swap (load1_stmt, load2_stmt);
7728 162 : std::swap (load1_stmt_info, load2_stmt_info);
7729 : }
7730 512 : if (!scan_operand_equal_p (gimple_assign_lhs (stmt1),
7731 : gimple_assign_rhs1 (load1_stmt)))
7732 0 : goto fail;
7733 :
7734 512 : tree var3 = NULL_TREE;
7735 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 3
7736 512 : && !scan_operand_equal_p (gimple_assign_lhs (stmt2),
7737 : gimple_assign_rhs1 (load2_stmt)))
7738 0 : goto fail;
7739 512 : else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7740 : {
7741 252 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7742 252 : if (TREE_CODE (DR_BASE_ADDRESS (load2_dr_info->dr)) != ADDR_EXPR
7743 252 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0)))
7744 0 : goto fail;
7745 252 : var3 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7746 252 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var3))
7747 252 : || lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var3))
7748 504 : || lookup_attribute ("omp simd inscan exclusive",
7749 252 : DECL_ATTRIBUTES (var3)))
7750 0 : goto fail;
7751 : }
7752 :
7753 512 : dr_vec_info *other_dr_info = STMT_VINFO_DR_INFO (other_store_stmt_info);
7754 512 : if (TREE_CODE (DR_BASE_ADDRESS (other_dr_info->dr)) != ADDR_EXPR
7755 512 : || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0)))
7756 0 : goto fail;
7757 :
7758 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7759 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (other_dr_info->dr), 0);
7760 512 : if (!lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var1))
7761 512 : || !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var2))
7762 1024 : || (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7763 512 : == (!lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var2))))
7764 0 : goto fail;
7765 :
7766 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7767 256 : std::swap (var1, var2);
7768 :
7769 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
7770 : {
7771 252 : if (!lookup_attribute ("omp simd inscan exclusive",
7772 252 : DECL_ATTRIBUTES (var1)))
7773 0 : goto fail;
7774 252 : var1 = var3;
7775 : }
7776 :
7777 512 : if (loop_vinfo->scan_map == NULL)
7778 0 : goto fail;
7779 512 : tree *init = loop_vinfo->scan_map->get (var1);
7780 512 : if (init == NULL)
7781 0 : goto fail;
7782 :
7783 : /* The IL is as expected, now check if we can actually vectorize it.
7784 : Inclusive scan:
7785 : _26 = D.2043[_25];
7786 : _27 = D.2042[_25];
7787 : _28 = _26 + _27;
7788 : D.2043[_25] = _28;
7789 : D.2042[_25] = _28;
7790 : should be vectorized as (where _40 is the vectorized rhs
7791 : from the D.2042[_21] = 0; store):
7792 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7793 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7794 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7795 : _33 = _31 + _32;
7796 : // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7797 : _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7798 : _35 = _33 + _34;
7799 : // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7800 : // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7801 : _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7802 : _37 = _35 + _36;
7803 : // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7804 : // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7805 : _38 = _30 + _37;
7806 : _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7807 : MEM <vector(8) int> [(int *)&D.2043] = _39;
7808 : MEM <vector(8) int> [(int *)&D.2042] = _38;
7809 : Exclusive scan:
7810 : _26 = D.2043[_25];
7811 : D.2044[_25] = _26;
7812 : _27 = D.2042[_25];
7813 : _28 = _26 + _27;
7814 : D.2043[_25] = _28;
7815 : should be vectorized as (where _40 is the vectorized rhs
7816 : from the D.2042[_21] = 0; store):
7817 : _30 = MEM <vector(8) int> [(int *)&D.2043];
7818 : _31 = MEM <vector(8) int> [(int *)&D.2042];
7819 : _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7820 : _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7821 : _34 = _32 + _33;
7822 : // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7823 : // _31[3]+_31[4], ... _31[5]+.._31[6] };
7824 : _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7825 : _36 = _34 + _35;
7826 : // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7827 : // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7828 : _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7829 : _38 = _36 + _37;
7830 : // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7831 : // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7832 : _39 = _30 + _38;
7833 : _50 = _31 + _39;
7834 : _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7835 : MEM <vector(8) int> [(int *)&D.2044] = _39;
7836 : MEM <vector(8) int> [(int *)&D.2042] = _51; */
7837 512 : enum machine_mode vec_mode = TYPE_MODE (vectype);
7838 512 : optab optab = optab_for_tree_code (code, vectype, optab_default);
7839 512 : if (!optab || !can_implement_p (optab, vec_mode))
7840 0 : goto fail;
7841 :
7842 512 : int units_log2 = scan_store_can_perm_p (vectype, *init);
7843 512 : if (units_log2 == -1)
7844 0 : goto fail;
7845 :
7846 : return true;
7847 : }
7848 :
7849 :
7850 : /* Function vectorizable_scan_store.
7851 :
7852 : Helper of vectorizable_score, arguments like on vectorizable_store.
7853 : Handle only the transformation, checking is done in check_scan_store. */
7854 :
7855 : static bool
7856 512 : vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
7857 : slp_tree slp_node, gimple_stmt_iterator *gsi)
7858 : {
7859 512 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7860 512 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
7861 512 : tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
7862 512 : tree vectype = SLP_TREE_VECTYPE (slp_node);
7863 :
7864 512 : if (dump_enabled_p ())
7865 492 : dump_printf_loc (MSG_NOTE, vect_location,
7866 : "transform scan store.\n");
7867 :
7868 512 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7869 512 : tree rhs = gimple_assign_rhs1 (stmt);
7870 512 : gcc_assert (TREE_CODE (rhs) == SSA_NAME);
7871 :
7872 512 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7873 512 : bool inscan_var_store
7874 512 : = lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)) != NULL;
7875 :
7876 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7877 : {
7878 126 : use_operand_p use_p;
7879 126 : imm_use_iterator iter;
7880 126 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs)
7881 : {
7882 126 : gimple *use_stmt = USE_STMT (use_p);
7883 126 : if (use_stmt == stmt || is_gimple_debug (use_stmt))
7884 0 : continue;
7885 126 : rhs = gimple_assign_lhs (use_stmt);
7886 126 : break;
7887 126 : }
7888 : }
7889 :
7890 512 : gimple *def_stmt = SSA_NAME_DEF_STMT (rhs);
7891 512 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
7892 512 : if (code == POINTER_PLUS_EXPR)
7893 0 : code = PLUS_EXPR;
7894 512 : gcc_assert (TREE_CODE_LENGTH (code) == binary_op
7895 : && commutative_tree_code (code));
7896 512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
7897 512 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
7898 512 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == SSA_NAME);
7899 512 : gimple *load1_stmt = SSA_NAME_DEF_STMT (rhs1);
7900 512 : gimple *load2_stmt = SSA_NAME_DEF_STMT (rhs2);
7901 512 : stmt_vec_info load1_stmt_info = loop_vinfo->lookup_stmt (load1_stmt);
7902 512 : stmt_vec_info load2_stmt_info = loop_vinfo->lookup_stmt (load2_stmt);
7903 512 : dr_vec_info *load1_dr_info = STMT_VINFO_DR_INFO (load1_stmt_info);
7904 512 : dr_vec_info *load2_dr_info = STMT_VINFO_DR_INFO (load2_stmt_info);
7905 512 : tree var1 = TREE_OPERAND (DR_BASE_ADDRESS (load1_dr_info->dr), 0);
7906 512 : tree var2 = TREE_OPERAND (DR_BASE_ADDRESS (load2_dr_info->dr), 0);
7907 :
7908 512 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var1)))
7909 : {
7910 436 : std::swap (rhs1, rhs2);
7911 436 : std::swap (var1, var2);
7912 436 : std::swap (load1_dr_info, load2_dr_info);
7913 : }
7914 :
7915 512 : tree *init = loop_vinfo->scan_map->get (var1);
7916 512 : gcc_assert (init);
7917 :
7918 512 : unsigned HOST_WIDE_INT nunits;
7919 512 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
7920 : gcc_unreachable ();
7921 512 : auto_vec<enum scan_store_kind, 16> use_whole_vector;
7922 512 : int units_log2 = scan_store_can_perm_p (vectype, *init, &use_whole_vector);
7923 512 : gcc_assert (units_log2 > 0);
7924 512 : auto_vec<tree, 16> perms;
7925 512 : perms.quick_grow (units_log2 + 1);
7926 512 : tree zero_vec = NULL_TREE, masktype = NULL_TREE;
7927 2904 : for (int i = 0; i <= units_log2; ++i)
7928 : {
7929 1880 : unsigned HOST_WIDE_INT j, k;
7930 1880 : vec_perm_builder sel (nunits, nunits, 1);
7931 1880 : sel.quick_grow (nunits);
7932 1880 : if (i == units_log2)
7933 4864 : for (j = 0; j < nunits; ++j)
7934 4352 : sel[j] = nunits - 1;
7935 : else
7936 : {
7937 5208 : for (j = 0; j < (HOST_WIDE_INT_1U << i); ++j)
7938 3840 : sel[j] = j;
7939 13208 : for (k = 0; j < nunits; ++j, ++k)
7940 11840 : sel[j] = nunits + k;
7941 : }
7942 3248 : vec_perm_indices indices (sel, i == units_log2 ? 1 : 2, nunits);
7943 1880 : if (!use_whole_vector.is_empty ()
7944 0 : && use_whole_vector[i] != scan_store_kind_perm)
7945 : {
7946 0 : if (zero_vec == NULL_TREE)
7947 0 : zero_vec = build_zero_cst (vectype);
7948 0 : if (masktype == NULL_TREE
7949 0 : && use_whole_vector[i] == scan_store_kind_lshift_cond)
7950 0 : masktype = truth_type_for (vectype);
7951 0 : perms[i] = vect_gen_perm_mask_any (vectype, indices);
7952 : }
7953 : else
7954 1880 : perms[i] = vect_gen_perm_mask_checked (vectype, indices);
7955 1880 : }
7956 :
7957 512 : tree vec_oprnd1 = NULL_TREE;
7958 512 : tree vec_oprnd2 = NULL_TREE;
7959 512 : tree vec_oprnd3 = NULL_TREE;
7960 512 : tree dataref_ptr = DR_BASE_ADDRESS (dr_info->dr);
7961 512 : tree dataref_offset = build_int_cst (ref_type, 0);
7962 512 : tree bump = vect_get_data_ptr_bump (vinfo, dr_info, vectype, VMAT_CONTIGUOUS);
7963 512 : tree ldataref_ptr = NULL_TREE;
7964 512 : tree orig = NULL_TREE;
7965 512 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4 && !inscan_var_store)
7966 126 : ldataref_ptr = DR_BASE_ADDRESS (load1_dr_info->dr);
7967 : /* The initialization is invariant. */
7968 512 : vec_oprnd1 = vect_init_vector (vinfo, stmt_info, *init, vectype, NULL);
7969 512 : auto_vec<tree> vec_oprnds2;
7970 512 : auto_vec<tree> vec_oprnds3;
7971 512 : if (ldataref_ptr == NULL)
7972 : {
7973 : /* We want to lookup the vector operands of the reduction, not those
7974 : of the store - for SLP we have to use the proper SLP node for the
7975 : lookup, which should be the single child of the scan store. */
7976 386 : vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
7977 : rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
7978 : /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
7979 : them in SLP child order. So we have to swap here with logic
7980 : similar to above. */
7981 386 : stmt_vec_info load
7982 386 : = SLP_TREE_SCALAR_STMTS (SLP_TREE_CHILDREN
7983 386 : (SLP_TREE_CHILDREN (slp_node)[0])[0])[0];
7984 386 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (load);
7985 386 : tree var = TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0);
7986 386 : if (lookup_attribute ("omp simd inscan", DECL_ATTRIBUTES (var)))
7987 820 : for (unsigned i = 0; i < vec_oprnds2.length (); ++i)
7988 494 : std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
7989 : }
7990 : else
7991 126 : vect_get_vec_defs (vinfo, slp_node,
7992 : rhs2, &vec_oprnds3);
7993 1248 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
7994 : {
7995 736 : if (ldataref_ptr == NULL)
7996 554 : vec_oprnd2 = vec_oprnds2[j];
7997 736 : vec_oprnd3 = vec_oprnds3[j];
7998 736 : if (j == 0)
7999 : orig = vec_oprnd3;
8000 224 : else if (!inscan_var_store)
8001 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8002 :
8003 736 : if (ldataref_ptr)
8004 : {
8005 182 : vec_oprnd2 = make_ssa_name (vectype);
8006 182 : tree data_ref = fold_build2 (MEM_REF, vectype,
8007 : unshare_expr (ldataref_ptr),
8008 : dataref_offset);
8009 182 : vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
8010 182 : gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
8011 182 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8012 : }
8013 :
8014 736 : tree v = vec_oprnd2;
8015 3068 : for (int i = 0; i < units_log2; ++i)
8016 : {
8017 2332 : tree new_temp = make_ssa_name (vectype);
8018 2332 : gimple *g = gimple_build_assign (new_temp, VEC_PERM_EXPR,
8019 : (zero_vec
8020 0 : && (use_whole_vector[i]
8021 0 : != scan_store_kind_perm))
8022 : ? zero_vec : vec_oprnd1, v,
8023 2332 : perms[i]);
8024 2332 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8025 :
8026 2332 : if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
8027 : {
8028 : /* Whole vector shift shifted in zero bits, but if *init
8029 : is not initializer_zerop, we need to replace those elements
8030 : with elements from vec_oprnd1. */
8031 0 : tree_vector_builder vb (masktype, nunits, 1);
8032 0 : for (unsigned HOST_WIDE_INT k = 0; k < nunits; ++k)
8033 0 : vb.quick_push (k < (HOST_WIDE_INT_1U << i)
8034 : ? boolean_false_node : boolean_true_node);
8035 :
8036 0 : tree new_temp2 = make_ssa_name (vectype);
8037 0 : g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (),
8038 : new_temp, vec_oprnd1);
8039 0 : vect_finish_stmt_generation (vinfo, stmt_info,
8040 : g, gsi);
8041 0 : new_temp = new_temp2;
8042 0 : }
8043 :
8044 : /* For exclusive scan, perform the perms[i] permutation once
8045 : more. */
8046 2332 : if (i == 0
8047 1100 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4
8048 728 : && v == vec_oprnd2)
8049 : {
8050 364 : v = new_temp;
8051 364 : --i;
8052 364 : continue;
8053 : }
8054 :
8055 1968 : tree new_temp2 = make_ssa_name (vectype);
8056 1968 : g = gimple_build_assign (new_temp2, code, v, new_temp);
8057 1968 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8058 :
8059 1968 : v = new_temp2;
8060 : }
8061 :
8062 736 : tree new_temp = make_ssa_name (vectype);
8063 736 : gimple *g = gimple_build_assign (new_temp, code, orig, v);
8064 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8065 :
8066 736 : tree last_perm_arg = new_temp;
8067 : /* For exclusive scan, new_temp computed above is the exclusive scan
8068 : prefix sum. Turn it into inclusive prefix sum for the broadcast
8069 : of the last element into orig. */
8070 736 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) == 4)
8071 : {
8072 364 : last_perm_arg = make_ssa_name (vectype);
8073 364 : g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
8074 364 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8075 : }
8076 :
8077 736 : orig = make_ssa_name (vectype);
8078 2208 : g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
8079 736 : last_perm_arg, perms[units_log2]);
8080 736 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8081 :
8082 736 : if (!inscan_var_store)
8083 : {
8084 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8085 : unshare_expr (dataref_ptr),
8086 : dataref_offset);
8087 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8088 368 : g = gimple_build_assign (data_ref, new_temp);
8089 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8090 : }
8091 : }
8092 :
8093 512 : if (inscan_var_store)
8094 624 : for (unsigned j = 0; j < vec_oprnds3.length (); j++)
8095 : {
8096 368 : if (j != 0)
8097 112 : dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
8098 :
8099 368 : tree data_ref = fold_build2 (MEM_REF, vectype,
8100 : unshare_expr (dataref_ptr),
8101 : dataref_offset);
8102 368 : vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
8103 368 : gimple *g = gimple_build_assign (data_ref, orig);
8104 368 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
8105 : }
8106 512 : return true;
8107 512 : }
8108 :
8109 :
8110 : /* Function vectorizable_store.
8111 :
8112 : Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8113 : that can be vectorized.
8114 : If COST_VEC is passed, calculate costs but don't change anything,
8115 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
8116 : it, and insert it at GSI.
8117 : Return true if STMT_INFO is vectorizable in this way. */
8118 :
8119 : static bool
8120 2150672 : vectorizable_store (vec_info *vinfo,
8121 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
8122 : slp_tree slp_node,
8123 : stmt_vector_for_cost *cost_vec)
8124 : {
8125 2150672 : tree data_ref;
8126 2150672 : tree vec_oprnd = NULL_TREE;
8127 2150672 : tree elem_type;
8128 2150672 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
8129 2150672 : class loop *loop = NULL;
8130 2150672 : machine_mode vec_mode;
8131 2150672 : tree dummy;
8132 2150672 : enum vect_def_type rhs_dt = vect_unknown_def_type;
8133 2150672 : enum vect_def_type mask_dt = vect_unknown_def_type;
8134 2150672 : tree dataref_ptr = NULL_TREE;
8135 2150672 : tree dataref_offset = NULL_TREE;
8136 2150672 : int j;
8137 2150672 : stmt_vec_info first_stmt_info;
8138 2150672 : bool grouped_store;
8139 2150672 : unsigned int group_size, i;
8140 2150672 : unsigned int vec_num;
8141 2150672 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
8142 2150672 : tree aggr_type;
8143 2150672 : poly_uint64 vf;
8144 2150672 : vec_load_store_type vls_type;
8145 2150672 : tree ref_type;
8146 :
8147 2150672 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8148 : return false;
8149 :
8150 2150672 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8151 244891 : && cost_vec)
8152 : return false;
8153 :
8154 : /* Is vectorizable store? */
8155 :
8156 1905781 : tree mask_vectype = NULL_TREE;
8157 1905781 : slp_tree mask_node = NULL;
8158 1905781 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
8159 : {
8160 1830974 : tree scalar_dest = gimple_assign_lhs (assign);
8161 1830974 : if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
8162 1830974 : && is_pattern_stmt_p (stmt_info))
8163 1679 : scalar_dest = TREE_OPERAND (scalar_dest, 0);
8164 1830974 : if (TREE_CODE (scalar_dest) != ARRAY_REF
8165 1830974 : && TREE_CODE (scalar_dest) != BIT_FIELD_REF
8166 : && TREE_CODE (scalar_dest) != INDIRECT_REF
8167 : && TREE_CODE (scalar_dest) != COMPONENT_REF
8168 : && TREE_CODE (scalar_dest) != IMAGPART_EXPR
8169 : && TREE_CODE (scalar_dest) != REALPART_EXPR
8170 : && TREE_CODE (scalar_dest) != MEM_REF)
8171 : return false;
8172 : }
8173 : else
8174 : {
8175 74807 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
8176 13276 : if (!call || !gimple_call_internal_p (call))
8177 : return false;
8178 :
8179 8091 : internal_fn ifn = gimple_call_internal_fn (call);
8180 8091 : if (!internal_store_fn_p (ifn))
8181 : return false;
8182 :
8183 1857 : int mask_index = internal_fn_mask_index (ifn);
8184 1857 : if (mask_index >= 0)
8185 1857 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
8186 1857 : if (mask_index >= 0
8187 1857 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
8188 : &mask_node, &mask_dt,
8189 : &mask_vectype))
8190 : return false;
8191 : }
8192 :
8193 1388604 : tree vectype = SLP_TREE_VECTYPE (slp_node), rhs_vectype = NULL_TREE;
8194 1388604 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8195 :
8196 1388604 : if (loop_vinfo)
8197 : {
8198 229116 : loop = LOOP_VINFO_LOOP (loop_vinfo);
8199 229116 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8200 : }
8201 : else
8202 : vf = 1;
8203 1388604 : vec_num = vect_get_num_copies (vinfo, slp_node);
8204 :
8205 : /* FORNOW. This restriction should be relaxed. */
8206 1388604 : if (loop
8207 1388891 : && nested_in_vect_loop_p (loop, stmt_info)
8208 1388899 : && vec_num > 1)
8209 : {
8210 8 : if (dump_enabled_p ())
8211 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8212 : "multiple types in nested loop.\n");
8213 : return false;
8214 : }
8215 :
8216 1388596 : slp_tree op_node;
8217 1388596 : if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
8218 : &op_node, &rhs_dt, &rhs_vectype, &vls_type))
8219 : return false;
8220 :
8221 1388572 : elem_type = TREE_TYPE (vectype);
8222 1388572 : vec_mode = TYPE_MODE (vectype);
8223 :
8224 1388572 : if (!STMT_VINFO_DATA_REF (stmt_info))
8225 : return false;
8226 :
8227 1388572 : vect_load_store_data _ls_data{};
8228 1388572 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
8229 1388572 : if (cost_vec
8230 1388572 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
8231 : vls_type, &_ls_data))
8232 : return false;
8233 : /* Temporary aliases to analysis data, should not be modified through
8234 : these. */
8235 1387948 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
8236 1387948 : const dr_alignment_support alignment_support_scheme
8237 : = ls.alignment_support_scheme;
8238 1387948 : const int misalignment = ls.misalignment;
8239 1387948 : const poly_int64 poffset = ls.poffset;
8240 :
8241 1387948 : if (slp_node->ldst_lanes
8242 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
8243 : {
8244 0 : if (dump_enabled_p ())
8245 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8246 : "discovered store-lane but cannot use it.\n");
8247 : return false;
8248 : }
8249 :
8250 1387948 : if (mask_node)
8251 : {
8252 1767 : if (memory_access_type == VMAT_CONTIGUOUS)
8253 : {
8254 616 : if (!VECTOR_MODE_P (vec_mode)
8255 3018 : || !can_vec_mask_load_store_p (vec_mode,
8256 1509 : TYPE_MODE (mask_vectype), false))
8257 : return false;
8258 : }
8259 258 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
8260 258 : && (!mat_gather_scatter_p (memory_access_type)
8261 242 : || (memory_access_type == VMAT_GATHER_SCATTER_LEGACY
8262 170 : && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
8263 : {
8264 16 : if (dump_enabled_p ())
8265 16 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8266 : "unsupported access type for masked store.\n");
8267 : return false;
8268 : }
8269 242 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
8270 : {
8271 72 : if (dump_enabled_p ())
8272 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8273 : "unsupported masked emulated scatter.\n");
8274 : return false;
8275 : }
8276 : }
8277 : else
8278 : {
8279 : /* FORNOW. In some cases can vectorize even if data-type not supported
8280 : (e.g. - array initialization with 0). */
8281 1386181 : if (!can_implement_p (mov_optab, vec_mode))
8282 : return false;
8283 : }
8284 :
8285 1387746 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
8286 1387746 : grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
8287 2567310 : && !mat_gather_scatter_p (memory_access_type));
8288 1179564 : if (grouped_store)
8289 : {
8290 1179564 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8291 1179564 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8292 1179564 : group_size = DR_GROUP_SIZE (first_stmt_info);
8293 : }
8294 : else
8295 : {
8296 1387746 : first_stmt_info = stmt_info;
8297 1387746 : first_dr_info = dr_info;
8298 : group_size = 1;
8299 : }
8300 :
8301 1387746 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
8302 : {
8303 1076 : if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
8304 : mask_node, memory_access_type))
8305 : return false;
8306 : }
8307 :
8308 1944379 : bool costing_p = cost_vec;
8309 1386978 : if (costing_p) /* transformation not required. */
8310 : {
8311 830345 : if (loop_vinfo
8312 164719 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
8313 77435 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
8314 : vls_type, group_size, &ls,
8315 : mask_node);
8316 :
8317 830345 : if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
8318 830345 : || (mask_node
8319 1038 : && !vect_maybe_update_slp_op_vectype (mask_node,
8320 : mask_vectype)))
8321 : {
8322 0 : if (dump_enabled_p ())
8323 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8324 : "incompatible vector types for invariants\n");
8325 : return false;
8326 : }
8327 :
8328 830345 : if (dump_enabled_p ()
8329 : && memory_access_type != VMAT_ELEMENTWISE
8330 15285 : && memory_access_type != VMAT_STRIDED_SLP
8331 14621 : && memory_access_type != VMAT_INVARIANT
8332 844966 : && alignment_support_scheme != dr_aligned)
8333 5128 : dump_printf_loc (MSG_NOTE, vect_location,
8334 : "Vectorizing an unaligned access.\n");
8335 : }
8336 :
8337 : /* Transform. */
8338 :
8339 : if (!costing_p)
8340 557401 : ensure_base_align (dr_info);
8341 :
8342 1387746 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
8343 : {
8344 1024 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
8345 1024 : gcc_assert (SLP_TREE_LANES (slp_node) == 1);
8346 1024 : if (costing_p)
8347 : {
8348 512 : unsigned int inside_cost = 0, prologue_cost = 0;
8349 512 : if (vls_type == VLS_STORE_INVARIANT)
8350 0 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8351 : slp_node, 0, vect_prologue);
8352 512 : vect_get_store_cost (vinfo, stmt_info, slp_node, 1,
8353 : alignment_support_scheme, misalignment,
8354 : &inside_cost, cost_vec);
8355 :
8356 512 : if (dump_enabled_p ())
8357 492 : dump_printf_loc (MSG_NOTE, vect_location,
8358 : "vect_model_store_cost: inside_cost = %d, "
8359 : "prologue_cost = %d .\n",
8360 : inside_cost, prologue_cost);
8361 :
8362 512 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8363 512 : slp_node->data = new vect_load_store_data (std::move (ls));
8364 :
8365 512 : return true;
8366 : }
8367 512 : return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
8368 : }
8369 :
8370 : /* FORNOW */
8371 1386722 : gcc_assert (!grouped_store
8372 : || !loop
8373 : || !nested_in_vect_loop_p (loop, stmt_info));
8374 :
8375 1386722 : grouped_store = false;
8376 1386722 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
8377 1386722 : gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
8378 : || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
8379 1386722 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
8380 :
8381 1386722 : ref_type = get_group_alias_ptr_type (first_stmt_info);
8382 :
8383 1386722 : if (!costing_p && dump_enabled_p ())
8384 12418 : dump_printf_loc (MSG_NOTE, vect_location, "transform store.\n");
8385 :
8386 1386722 : if (memory_access_type == VMAT_ELEMENTWISE
8387 1386722 : || memory_access_type == VMAT_STRIDED_SLP)
8388 : {
8389 29356 : unsigned inside_cost = 0, prologue_cost = 0;
8390 29356 : gimple_stmt_iterator incr_gsi;
8391 29356 : bool insert_after;
8392 29356 : tree offvar = NULL_TREE;
8393 29356 : tree ivstep;
8394 29356 : tree running_off;
8395 29356 : tree stride_base, stride_step, alias_off;
8396 29356 : tree vec_oprnd = NULL_TREE;
8397 29356 : tree dr_offset;
8398 : /* Checked by get_load_store_type. */
8399 29356 : unsigned int const_nunits = nunits.to_constant ();
8400 :
8401 29356 : gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
8402 29356 : gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
8403 :
8404 29356 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
8405 29356 : stride_base
8406 29356 : = fold_build_pointer_plus
8407 : (DR_BASE_ADDRESS (first_dr_info->dr),
8408 : size_binop (PLUS_EXPR,
8409 : convert_to_ptrofftype (dr_offset),
8410 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
8411 29356 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
8412 :
8413 : /* For a store with loop-invariant (but other than power-of-2)
8414 : stride (i.e. not a grouped access) like so:
8415 :
8416 : for (i = 0; i < n; i += stride)
8417 : array[i] = ...;
8418 :
8419 : we generate a new induction variable and new stores from
8420 : the components of the (vectorized) rhs:
8421 :
8422 : for (j = 0; ; j += VF*stride)
8423 : vectemp = ...;
8424 : tmp1 = vectemp[0];
8425 : array[j] = tmp1;
8426 : tmp2 = vectemp[1];
8427 : array[j + stride] = tmp2;
8428 : ...
8429 : */
8430 :
8431 : /* ??? Modify local copies of alignment_support_scheme and
8432 : misalignment, but this part of analysis should be done
8433 : earlier and remembered, likewise the chosen load mode. */
8434 29356 : const dr_alignment_support tem = alignment_support_scheme;
8435 29356 : dr_alignment_support alignment_support_scheme = tem;
8436 29356 : const int tem2 = misalignment;
8437 29356 : int misalignment = tem2;
8438 :
8439 29356 : unsigned nstores = const_nunits;
8440 29356 : unsigned lnel = 1;
8441 29356 : tree ltype = elem_type;
8442 29356 : tree lvectype = vectype;
8443 29356 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
8444 29356 : if (n == const_nunits)
8445 : {
8446 2945 : int mis_align = dr_misalignment (first_dr_info, vectype);
8447 : /* With VF > 1 we advance the DR by step, if that is constant
8448 : and only aligned when performed VF times, DR alignment
8449 : analysis can analyze this as aligned since it assumes
8450 : contiguous accesses. But that is not how we code generate
8451 : here, so adjust for this. */
8452 2945 : if (maybe_gt (vf, 1u)
8453 4476 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8454 4241 : DR_TARGET_ALIGNMENT (first_dr_info)))
8455 235 : mis_align = -1;
8456 2945 : dr_alignment_support dr_align
8457 2945 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
8458 : mis_align);
8459 2945 : if (dr_align == dr_aligned
8460 2945 : || dr_align == dr_unaligned_supported)
8461 : {
8462 29356 : nstores = 1;
8463 29356 : lnel = const_nunits;
8464 29356 : ltype = vectype;
8465 29356 : lvectype = vectype;
8466 29356 : alignment_support_scheme = dr_align;
8467 29356 : misalignment = mis_align;
8468 : }
8469 : }
8470 26411 : else if (n > 1)
8471 : {
8472 1971 : nstores = const_nunits / n;
8473 1971 : lnel = n;
8474 1971 : ltype = build_vector_type (elem_type, n);
8475 1971 : lvectype = vectype;
8476 1971 : int mis_align = dr_misalignment (first_dr_info, ltype);
8477 1971 : if (maybe_gt (vf, 1u)
8478 3942 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
8479 3296 : DR_TARGET_ALIGNMENT (first_dr_info)))
8480 646 : mis_align = -1;
8481 1971 : dr_alignment_support dr_align
8482 1971 : = vect_supportable_dr_alignment (vinfo, dr_info, ltype,
8483 : mis_align);
8484 1971 : alignment_support_scheme = dr_align;
8485 1971 : misalignment = mis_align;
8486 :
8487 : /* First check if vec_extract optab doesn't support extraction
8488 : of vector elts directly. */
8489 1971 : scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
8490 1971 : machine_mode vmode;
8491 3942 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8492 3748 : || !related_vector_mode (TYPE_MODE (vectype), elmode,
8493 2165 : n).exists (&vmode)
8494 1777 : || (convert_optab_handler (vec_extract_optab,
8495 1777 : TYPE_MODE (vectype), vmode)
8496 : == CODE_FOR_nothing)
8497 1971 : || !(dr_align == dr_aligned
8498 172 : || dr_align == dr_unaligned_supported))
8499 : {
8500 : /* Try to avoid emitting an extract of vector elements
8501 : by performing the extracts using an integer type of the
8502 : same size, extracting from a vector of those and then
8503 : re-interpreting it as the original vector type if
8504 : supported. */
8505 1799 : unsigned lsize = n * GET_MODE_BITSIZE (elmode);
8506 1799 : unsigned int lnunits = const_nunits / n;
8507 : /* If we can't construct such a vector fall back to
8508 : element extracts from the original vector type and
8509 : element size stores. */
8510 1799 : if (int_mode_for_size (lsize, 0).exists (&elmode)
8511 1799 : && VECTOR_MODE_P (TYPE_MODE (vectype))
8512 1799 : && related_vector_mode (TYPE_MODE (vectype), elmode,
8513 1799 : lnunits).exists (&vmode)
8514 1771 : && (convert_optab_handler (vec_extract_optab,
8515 : vmode, elmode)
8516 : != CODE_FOR_nothing))
8517 : {
8518 1771 : nstores = lnunits;
8519 1771 : lnel = n;
8520 1771 : ltype = build_nonstandard_integer_type (lsize, 1);
8521 1771 : lvectype = build_vector_type (ltype, nstores);
8522 : }
8523 : /* Else fall back to vector extraction anyway.
8524 : Fewer stores are more important than avoiding spilling
8525 : of the vector we extract from. Compared to the
8526 : construction case in vectorizable_load no store-forwarding
8527 : issue exists here for reasonable archs. But only
8528 : if the store is supported. */
8529 28 : else if (!(dr_align == dr_aligned
8530 28 : || dr_align == dr_unaligned_supported))
8531 : {
8532 29356 : nstores = const_nunits;
8533 29356 : lnel = 1;
8534 29356 : ltype = elem_type;
8535 29356 : lvectype = vectype;
8536 : }
8537 : }
8538 : }
8539 :
8540 29356 : if (costing_p)
8541 : {
8542 : /* Record the decomposition type for target access during costing. */
8543 25993 : ls.ls_type = lvectype;
8544 25993 : ls.ls_eltype = ltype;
8545 : }
8546 : else
8547 3363 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
8548 :
8549 29356 : unsigned align;
8550 29356 : if (alignment_support_scheme == dr_aligned)
8551 1241 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
8552 : else
8553 28115 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
8554 : /* Alignment is at most the access size if we do multiple stores. */
8555 29356 : if (nstores > 1)
8556 26411 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
8557 29356 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
8558 29356 : int ncopies = vec_num;
8559 :
8560 29356 : if (!costing_p)
8561 : {
8562 3363 : ivstep = stride_step;
8563 :
8564 3363 : tree increment = fold_convert (TREE_TYPE (ivstep),
8565 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
8566 :
8567 3363 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
8568 : increment);
8569 :
8570 3363 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
8571 :
8572 3363 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
8573 3363 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
8574 3363 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
8575 : else
8576 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
8577 : true, NULL_TREE, true,
8578 : GSI_SAME_STMT);
8579 :
8580 3363 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
8581 : insert_after, &offvar, NULL,
8582 3363 : LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
8583 :
8584 3363 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
8585 : }
8586 :
8587 29356 : alias_off = build_int_cst (ref_type, 0);
8588 29356 : auto_vec<tree> vec_oprnds;
8589 : /* For costing some adjacent vector stores, we'd like to cost with
8590 : the total number of them once instead of cost each one by one. */
8591 29356 : unsigned int n_adjacent_stores = 0;
8592 29356 : running_off = offvar;
8593 29356 : if (!costing_p)
8594 3363 : vect_get_slp_defs (op_node, &vec_oprnds);
8595 29356 : unsigned int group_el = 0;
8596 29356 : unsigned HOST_WIDE_INT elsz
8597 29356 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
8598 69949 : for (j = 0; j < ncopies; j++)
8599 : {
8600 40593 : if (!costing_p)
8601 : {
8602 5203 : vec_oprnd = vec_oprnds[j];
8603 : /* Pun the vector to extract from if necessary. */
8604 5203 : if (lvectype != vectype)
8605 : {
8606 1010 : tree tem = make_ssa_name (lvectype);
8607 1010 : tree cvt = build1 (VIEW_CONVERT_EXPR, lvectype, vec_oprnd);
8608 1010 : gimple *pun = gimple_build_assign (tem, cvt);
8609 1010 : vect_finish_stmt_generation (vinfo, stmt_info, pun, gsi);
8610 1010 : vec_oprnd = tem;
8611 : }
8612 : }
8613 180429 : for (i = 0; i < nstores; i++)
8614 : {
8615 139836 : if (costing_p)
8616 : {
8617 123985 : n_adjacent_stores++;
8618 123985 : continue;
8619 : }
8620 15851 : tree newref, newoff;
8621 15851 : gimple *incr, *assign;
8622 15851 : tree size = TYPE_SIZE (ltype);
8623 : /* Extract the i'th component. */
8624 15851 : tree pos = fold_build2 (MULT_EXPR, bitsizetype,
8625 : bitsize_int (i), size);
8626 15851 : tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
8627 : size, pos);
8628 :
8629 15851 : elem = force_gimple_operand_gsi (gsi, elem, true, NULL_TREE, true,
8630 : GSI_SAME_STMT);
8631 :
8632 15851 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
8633 15851 : group_el * elsz);
8634 15851 : newref = build2 (MEM_REF, ltype, running_off, this_off);
8635 15851 : vect_copy_ref_info (newref, DR_REF (first_dr_info->dr));
8636 :
8637 : /* And store it to *running_off. */
8638 15851 : assign = gimple_build_assign (newref, elem);
8639 15851 : vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi);
8640 :
8641 15851 : group_el += lnel;
8642 15851 : if (group_el == group_size)
8643 : {
8644 14190 : newoff = copy_ssa_name (running_off, NULL);
8645 14190 : incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
8646 : running_off, stride_step);
8647 14190 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
8648 :
8649 14190 : running_off = newoff;
8650 14190 : group_el = 0;
8651 : }
8652 : }
8653 : }
8654 :
8655 29356 : if (costing_p)
8656 : {
8657 25993 : if (n_adjacent_stores > 0)
8658 : {
8659 : /* Take a single lane vector type store as scalar
8660 : store to avoid ICE like 110776. */
8661 25993 : if (VECTOR_TYPE_P (ltype)
8662 25993 : && maybe_ne (TYPE_VECTOR_SUBPARTS (ltype), 1U))
8663 1618 : vect_get_store_cost (vinfo, stmt_info, slp_node,
8664 : n_adjacent_stores, alignment_support_scheme,
8665 : misalignment, &inside_cost, cost_vec);
8666 : else
8667 24375 : inside_cost
8668 24375 : += record_stmt_cost (cost_vec, n_adjacent_stores,
8669 : scalar_store, slp_node, 0, vect_body);
8670 : /* Only need vector deconstruction when there is more
8671 : than one store. */
8672 25993 : if (nstores > 1)
8673 23959 : inside_cost
8674 23959 : += record_stmt_cost (cost_vec, ncopies,
8675 : vec_deconstruct, slp_node, 0, vect_body);
8676 : }
8677 25993 : if (dump_enabled_p ())
8678 664 : dump_printf_loc (MSG_NOTE, vect_location,
8679 : "vect_model_store_cost: inside_cost = %d, "
8680 : "prologue_cost = %d .\n",
8681 : inside_cost, prologue_cost);
8682 :
8683 25993 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8684 25993 : slp_node->data = new vect_load_store_data (std::move (ls));
8685 : }
8686 :
8687 29356 : return true;
8688 29356 : }
8689 :
8690 1357366 : gcc_assert (alignment_support_scheme);
8691 1357366 : vec_loop_masks *loop_masks
8692 197878 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
8693 1357366 : ? &LOOP_VINFO_MASKS (loop_vinfo)
8694 12 : : NULL);
8695 12 : vec_loop_lens *loop_lens
8696 197878 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
8697 : ? &LOOP_VINFO_LENS (loop_vinfo)
8698 0 : : NULL);
8699 :
8700 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8701 : are some difference here. We cannot enable both the lens and masks
8702 : during transform but it is allowed during analysis.
8703 : Shouldn't go with length-based approach if fully masked. */
8704 1357366 : if (cost_vec == NULL)
8705 : /* The cost_vec is NULL during transform. */
8706 553526 : gcc_assert ((!loop_lens || !loop_masks));
8707 :
8708 : /* Targets with store-lane instructions must not require explicit
8709 : realignment. vect_supportable_dr_alignment always returns either
8710 : dr_aligned or dr_unaligned_supported for masked operations. */
8711 1357366 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
8712 : && !mask_node
8713 : && !loop_masks)
8714 : || alignment_support_scheme == dr_aligned
8715 : || alignment_support_scheme == dr_unaligned_supported);
8716 :
8717 1357366 : tree offset = NULL_TREE;
8718 1357366 : if (!known_eq (poffset, 0))
8719 4655 : offset = size_int (poffset);
8720 :
8721 1357366 : tree dr_increment;
8722 1357366 : tree dr_bump;
8723 :
8724 1357366 : tree vec_offset = NULL_TREE;
8725 1357366 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8726 : {
8727 1460 : aggr_type = NULL_TREE;
8728 1460 : dr_increment = NULL_TREE;
8729 1460 : dr_bump = NULL_TREE;
8730 : }
8731 1355906 : else if (mat_gather_scatter_p (memory_access_type))
8732 : {
8733 0 : aggr_type = elem_type;
8734 0 : if (!costing_p)
8735 : {
8736 0 : tree vtype = ls.ls_type ? ls.ls_type : vectype;
8737 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vtype,
8738 : ls.strided_offset_vectype,
8739 : loop_vinfo, gsi,
8740 : &dr_increment, &dr_bump,
8741 : &vec_offset);
8742 : }
8743 : }
8744 : else
8745 : {
8746 1355906 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8747 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
8748 : else
8749 : aggr_type = vectype;
8750 1355906 : if (!costing_p)
8751 : {
8752 553051 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
8753 : memory_access_type);
8754 553051 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
8755 : memory_access_type);
8756 : }
8757 : }
8758 :
8759 1357366 : if (loop_vinfo && mask_node && !costing_p)
8760 527 : LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
8761 :
8762 : /* In case the vectorization factor (VF) is bigger than the number
8763 : of elements that we can fit in a vectype (nunits), we have to generate
8764 : more than one vector stmt - i.e - we need to "unroll" the
8765 : vector stmt by a factor VF/nunits. */
8766 :
8767 1357366 : auto_vec<tree> dr_chain (group_size);
8768 1357366 : auto_vec<tree> vec_masks;
8769 1357366 : tree vec_mask = NULL;
8770 1357366 : auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
8771 6119482 : for (i = 0; i < group_size; i++)
8772 3404750 : gvec_oprnds.quick_push (new auto_vec<tree> ());
8773 :
8774 1357366 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
8775 : {
8776 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
8777 :
8778 0 : if (costing_p)
8779 : /* Update all incoming store operand nodes, the general handling
8780 : above only handles the mask and the first store operand node. */
8781 0 : for (slp_tree child : SLP_TREE_CHILDREN (slp_node))
8782 0 : if (child != mask_node
8783 0 : && !vect_maybe_update_slp_op_vectype (child, vectype))
8784 : {
8785 0 : if (dump_enabled_p ())
8786 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8787 : "incompatible vector types for invariants\n");
8788 : return false;
8789 : }
8790 0 : unsigned inside_cost = 0, prologue_cost = 0;
8791 : /* For costing some adjacent vector stores, we'd like to cost with
8792 : the total number of them once instead of cost each one by one. */
8793 0 : unsigned int n_adjacent_stores = 0;
8794 0 : int ncopies = vec_num / group_size;
8795 0 : for (j = 0; j < ncopies; j++)
8796 : {
8797 0 : if (j == 0)
8798 : {
8799 0 : if (!costing_p)
8800 : {
8801 0 : if (mask_node)
8802 : {
8803 0 : vect_get_slp_defs (mask_node, &vec_masks);
8804 0 : vec_mask = vec_masks[0];
8805 : }
8806 0 : dataref_ptr
8807 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8808 : aggr_type, NULL, offset, &dummy,
8809 : gsi, NULL, false, dr_increment);
8810 : }
8811 : }
8812 0 : else if (!costing_p)
8813 : {
8814 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8815 0 : if (mask_node)
8816 0 : vec_mask = vec_masks[j];
8817 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
8818 : stmt_info, dr_bump);
8819 : }
8820 :
8821 0 : if (costing_p)
8822 : {
8823 0 : n_adjacent_stores += group_size;
8824 0 : continue;
8825 : }
8826 :
8827 : /* Get an array into which we can store the individual vectors. */
8828 0 : tree vec_array = create_vector_array (vectype, group_size);
8829 :
8830 : /* Invalidate the current contents of VEC_ARRAY. This should
8831 : become an RTL clobber too, which prevents the vector registers
8832 : from being upward-exposed. */
8833 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8834 :
8835 : /* Store the individual vectors into the array. */
8836 0 : for (i = 0; i < group_size; i++)
8837 : {
8838 0 : slp_tree child;
8839 0 : if (i == 0 || !mask_node)
8840 0 : child = SLP_TREE_CHILDREN (slp_node)[i];
8841 : else
8842 0 : child = SLP_TREE_CHILDREN (slp_node)[i + 1];
8843 0 : vec_oprnd = SLP_TREE_VEC_DEFS (child)[j];
8844 0 : write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
8845 : i);
8846 : }
8847 :
8848 0 : tree final_mask = NULL;
8849 0 : tree final_len = NULL;
8850 0 : tree bias = NULL;
8851 0 : if (loop_masks)
8852 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
8853 : ncopies, vectype, j);
8854 0 : if (vec_mask)
8855 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
8856 : vec_mask, gsi);
8857 :
8858 0 : if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
8859 : {
8860 0 : if (loop_lens)
8861 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
8862 : ncopies, vectype, j, 1, true);
8863 : else
8864 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8865 0 : signed char biasval
8866 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
8867 0 : bias = build_int_cst (intQI_type_node, biasval);
8868 0 : if (!final_mask)
8869 : {
8870 0 : mask_vectype = truth_type_for (vectype);
8871 0 : final_mask = build_minus_one_cst (mask_vectype);
8872 : }
8873 : }
8874 :
8875 0 : gcall *call;
8876 0 : if (final_len && final_mask)
8877 : {
8878 : /* Emit:
8879 : MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8880 : LEN, BIAS, VEC_ARRAY). */
8881 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8882 0 : tree alias_ptr = build_int_cst (ref_type, align);
8883 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE_LANES, 6,
8884 : dataref_ptr, alias_ptr,
8885 : final_mask, final_len, bias,
8886 : vec_array);
8887 : }
8888 0 : else if (final_mask)
8889 : {
8890 : /* Emit:
8891 : MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8892 : VEC_ARRAY). */
8893 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
8894 0 : tree alias_ptr = build_int_cst (ref_type, align);
8895 0 : call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
8896 : dataref_ptr, alias_ptr,
8897 : final_mask, vec_array);
8898 : }
8899 : else
8900 : {
8901 : /* Emit:
8902 : MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8903 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8904 0 : call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
8905 0 : gimple_call_set_lhs (call, data_ref);
8906 : }
8907 0 : gimple_call_set_nothrow (call, true);
8908 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
8909 :
8910 : /* Record that VEC_ARRAY is now dead. */
8911 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
8912 : }
8913 :
8914 0 : if (costing_p)
8915 : {
8916 0 : if (n_adjacent_stores > 0)
8917 0 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
8918 : alignment_support_scheme, misalignment,
8919 : &inside_cost, cost_vec);
8920 0 : if (dump_enabled_p ())
8921 0 : dump_printf_loc (MSG_NOTE, vect_location,
8922 : "vect_model_store_cost: inside_cost = %d, "
8923 : "prologue_cost = %d .\n",
8924 : inside_cost, prologue_cost);
8925 :
8926 0 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
8927 0 : slp_node->data = new vect_load_store_data (std::move (ls));
8928 : }
8929 :
8930 : return true;
8931 : }
8932 :
8933 1357366 : if (mat_gather_scatter_p (memory_access_type))
8934 : {
8935 1460 : gcc_assert (!grouped_store || ls.ls_type);
8936 1460 : if (ls.ls_type)
8937 0 : vectype = ls.ls_type;
8938 1460 : auto_vec<tree> vec_offsets;
8939 1460 : unsigned int inside_cost = 0, prologue_cost = 0;
8940 1460 : int num_stmts = vec_num;
8941 3319 : for (j = 0; j < num_stmts; j++)
8942 : {
8943 1859 : gimple *new_stmt;
8944 1859 : if (j == 0)
8945 : {
8946 1460 : if (costing_p && vls_type == VLS_STORE_INVARIANT)
8947 202 : prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
8948 : slp_node, 0, vect_prologue);
8949 : else if (!costing_p)
8950 : {
8951 : /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8952 : DR_CHAIN is of size 1. */
8953 475 : gcc_assert (group_size == 1);
8954 475 : vect_get_slp_defs (op_node, gvec_oprnds[0]);
8955 475 : if (mask_node)
8956 70 : vect_get_slp_defs (mask_node, &vec_masks);
8957 :
8958 475 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8959 475 : vect_get_gather_scatter_ops (loop, slp_node,
8960 : &dataref_ptr, &vec_offsets);
8961 : else
8962 0 : dataref_ptr
8963 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info,
8964 : aggr_type, NULL, offset,
8965 : &dummy, gsi, NULL, false,
8966 : dr_increment);
8967 : }
8968 : }
8969 399 : else if (!costing_p)
8970 : {
8971 35 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
8972 35 : if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8973 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr,
8974 : gsi, stmt_info, dr_bump);
8975 : }
8976 :
8977 2571 : new_stmt = NULL;
8978 712 : if (!costing_p)
8979 : {
8980 510 : vec_oprnd = (*gvec_oprnds[0])[j];
8981 510 : if (mask_node)
8982 90 : vec_mask = vec_masks[j];
8983 : /* We should have caught mismatched types earlier. */
8984 510 : gcc_assert (ls.ls_type
8985 : || useless_type_conversion_p
8986 : (vectype, TREE_TYPE (vec_oprnd)));
8987 : }
8988 510 : tree final_mask = NULL_TREE;
8989 2369 : tree final_len = NULL_TREE;
8990 2369 : tree bias = NULL_TREE;
8991 510 : if (!costing_p)
8992 : {
8993 510 : if (loop_masks)
8994 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi,
8995 : loop_masks, num_stmts,
8996 : vectype, j);
8997 510 : if (vec_mask)
8998 90 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
8999 : final_mask, vec_mask, gsi);
9000 : }
9001 :
9002 1859 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
9003 1859 : tree alias_align_ptr = build_int_cst (ref_type, align);
9004 1859 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
9005 : {
9006 0 : if (costing_p)
9007 : {
9008 0 : if (ls.supported_offset_vectype
9009 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
9010 : vec_offset))
9011 0 : inside_cost
9012 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9013 : slp_node, 0, vect_body);
9014 0 : if (ls.supported_scale)
9015 0 : inside_cost
9016 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
9017 : slp_node, 0, vect_body);
9018 :
9019 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9020 0 : inside_cost
9021 0 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9022 : slp_node, 0, vect_body);
9023 0 : continue;
9024 0 : }
9025 :
9026 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
9027 0 : vec_offset = vec_offsets[j];
9028 :
9029 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9030 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
9031 :
9032 : /* Perform the offset conversion and scaling if necessary. */
9033 0 : if (!strided
9034 0 : && (ls.supported_offset_vectype || ls.supported_scale))
9035 : {
9036 0 : gimple_seq stmts = NULL;
9037 0 : if (ls.supported_offset_vectype)
9038 0 : vec_offset = gimple_convert
9039 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
9040 0 : if (ls.supported_scale)
9041 : {
9042 : /* Only scale the vec_offset if we haven't already. */
9043 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
9044 0 : || j == 0)
9045 : {
9046 0 : tree mult_cst = build_int_cst
9047 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
9048 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
9049 0 : tree mult = build_vector_from_val
9050 0 : (TREE_TYPE (vec_offset), mult_cst);
9051 0 : vec_offset = gimple_build
9052 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
9053 : vec_offset, mult);
9054 : }
9055 0 : scale = size_int (ls.supported_scale);
9056 : }
9057 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9058 : }
9059 :
9060 0 : if (ls.gs.ifn == IFN_MASK_LEN_SCATTER_STORE)
9061 : {
9062 0 : if (loop_lens)
9063 0 : final_len = vect_get_loop_len (loop_vinfo, gsi,
9064 : loop_lens, num_stmts,
9065 : vectype, j, 1, true);
9066 : else
9067 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9068 :
9069 0 : signed char biasval
9070 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9071 0 : bias = build_int_cst (intQI_type_node, biasval);
9072 0 : if (!final_mask)
9073 : {
9074 0 : mask_vectype = truth_type_for (vectype);
9075 0 : final_mask = build_minus_one_cst (mask_vectype);
9076 : }
9077 : }
9078 :
9079 0 : if (ls.ls_type)
9080 : {
9081 0 : gimple *conv_stmt
9082 0 : = gimple_build_assign (make_ssa_name (vectype),
9083 : VIEW_CONVERT_EXPR,
9084 : build1 (VIEW_CONVERT_EXPR, vectype,
9085 : vec_oprnd));
9086 0 : vect_finish_stmt_generation (vinfo, stmt_info, conv_stmt,
9087 : gsi);
9088 0 : vec_oprnd = gimple_get_lhs (conv_stmt);
9089 : }
9090 :
9091 0 : gcall *call;
9092 0 : if (final_len && final_mask)
9093 : {
9094 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
9095 0 : call = gimple_build_call_internal (
9096 : IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
9097 : alias_align_ptr,
9098 : vec_offset, scale, vec_oprnd, final_mask, final_len,
9099 : bias);
9100 : else
9101 : /* Non-vector offset indicates that prefer to take
9102 : MASK_LEN_STRIDED_STORE instead of the
9103 : IFN_MASK_SCATTER_STORE with direct stride arg.
9104 : Similar to the gather case we have checked the
9105 : alignment for a scatter already and assume
9106 : that the strided store has the same requirements. */
9107 0 : call = gimple_build_call_internal (
9108 : IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
9109 : vec_offset, vec_oprnd, final_mask, final_len, bias);
9110 : }
9111 0 : else if (final_mask)
9112 0 : call = gimple_build_call_internal
9113 0 : (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
9114 : alias_align_ptr,
9115 : vec_offset, scale, vec_oprnd, final_mask);
9116 : else
9117 0 : call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
9118 : dataref_ptr,
9119 : alias_align_ptr,
9120 : vec_offset,
9121 : scale, vec_oprnd);
9122 0 : gimple_call_set_nothrow (call, true);
9123 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9124 0 : new_stmt = call;
9125 : }
9126 1859 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
9127 : {
9128 : /* The builtin decls path for scatter is legacy, x86 only. */
9129 331 : gcc_assert (nunits.is_constant ()
9130 : && (!final_mask
9131 : || SCALAR_INT_MODE_P
9132 : (TYPE_MODE (TREE_TYPE (final_mask)))));
9133 331 : if (costing_p)
9134 : {
9135 200 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9136 200 : inside_cost
9137 200 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9138 : slp_node, 0, vect_body);
9139 200 : continue;
9140 200 : }
9141 :
9142 131 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9143 131 : poly_uint64 offset_nunits
9144 131 : = TYPE_VECTOR_SUBPARTS (offset_vectype);
9145 131 : if (known_eq (nunits, offset_nunits))
9146 : {
9147 55 : new_stmt = vect_build_one_scatter_store_call
9148 110 : (vinfo, stmt_info, slp_node, gsi,
9149 55 : ls.gs.decl, dataref_ptr, vec_offsets[j],
9150 : vec_oprnd, final_mask);
9151 55 : vect_finish_stmt_generation (vinfo, stmt_info,
9152 : new_stmt, gsi);
9153 : }
9154 76 : else if (known_eq (nunits, offset_nunits * 2))
9155 : {
9156 : /* We have a offset vector with half the number of
9157 : lanes but the builtins will store full vectype
9158 : data from the lower lanes. */
9159 30 : new_stmt = vect_build_one_scatter_store_call
9160 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9161 30 : dataref_ptr, vec_offsets[2 * j],
9162 : vec_oprnd, final_mask);
9163 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9164 : new_stmt, gsi);
9165 30 : int count = nunits.to_constant ();
9166 30 : vec_perm_builder sel (count, count, 1);
9167 30 : sel.quick_grow (count);
9168 412 : for (int i = 0; i < count; ++i)
9169 352 : sel[i] = i | (count / 2);
9170 30 : vec_perm_indices indices (sel, 2, count);
9171 30 : tree perm_mask
9172 30 : = vect_gen_perm_mask_checked (vectype, indices);
9173 30 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
9174 : vec_oprnd, vec_oprnd,
9175 : perm_mask);
9176 30 : vec_oprnd = make_ssa_name (vectype);
9177 30 : gimple_set_lhs (new_stmt, vec_oprnd);
9178 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9179 : new_stmt, gsi);
9180 30 : if (final_mask)
9181 : {
9182 20 : new_stmt = gimple_build_assign (NULL_TREE,
9183 : VEC_UNPACK_HI_EXPR,
9184 : final_mask);
9185 20 : final_mask = make_ssa_name
9186 20 : (truth_type_for (offset_vectype));
9187 20 : gimple_set_lhs (new_stmt, final_mask);
9188 20 : vect_finish_stmt_generation (vinfo, stmt_info,
9189 : new_stmt, gsi);
9190 : }
9191 :
9192 30 : new_stmt = vect_build_one_scatter_store_call
9193 60 : (vinfo, stmt_info, slp_node, gsi, ls.gs.decl,
9194 30 : dataref_ptr, vec_offsets[2 * j + 1],
9195 : vec_oprnd, final_mask);
9196 30 : vect_finish_stmt_generation (vinfo, stmt_info,
9197 : new_stmt, gsi);
9198 30 : }
9199 46 : else if (known_eq (nunits * 2, offset_nunits))
9200 : {
9201 : /* We have a offset vector with double the number of
9202 : lanes. Select the low/high part accordingly. */
9203 46 : vec_offset = vec_offsets[j / 2];
9204 46 : if (j & 1)
9205 : {
9206 23 : int count = offset_nunits.to_constant ();
9207 23 : vec_perm_builder sel (count, count, 1);
9208 23 : sel.quick_grow (count);
9209 286 : for (int i = 0; i < count; ++i)
9210 240 : sel[i] = i | (count / 2);
9211 23 : vec_perm_indices indices (sel, 2, count);
9212 23 : tree perm_mask = vect_gen_perm_mask_checked
9213 23 : (TREE_TYPE (vec_offset), indices);
9214 23 : new_stmt = gimple_build_assign (NULL_TREE,
9215 : VEC_PERM_EXPR,
9216 : vec_offset,
9217 : vec_offset,
9218 : perm_mask);
9219 23 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
9220 23 : gimple_set_lhs (new_stmt, vec_offset);
9221 23 : vect_finish_stmt_generation (vinfo, stmt_info,
9222 : new_stmt, gsi);
9223 23 : }
9224 :
9225 46 : new_stmt = vect_build_one_scatter_store_call
9226 46 : (vinfo, stmt_info, slp_node, gsi,
9227 : ls.gs.decl, dataref_ptr, vec_offset,
9228 : vec_oprnd, final_mask);
9229 46 : vect_finish_stmt_generation (vinfo, stmt_info,
9230 : new_stmt, gsi);
9231 : }
9232 : else
9233 0 : gcc_unreachable ();
9234 : }
9235 : else
9236 : {
9237 : /* Emulated scatter. */
9238 1528 : gcc_assert (!final_mask);
9239 1528 : if (costing_p)
9240 : {
9241 1149 : unsigned int cnunits = vect_nunits_for_cost (vectype);
9242 : /* For emulated scatter N offset vector element extracts
9243 : (we assume the scalar scaling and ptr + offset add is
9244 : consumed by the load). */
9245 1149 : inside_cost
9246 1149 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9247 : slp_node, 0, vect_body);
9248 : /* N scalar stores plus extracting the elements. */
9249 1149 : inside_cost
9250 1149 : += record_stmt_cost (cost_vec, 1, vec_deconstruct,
9251 : slp_node, 0, vect_body);
9252 1149 : inside_cost
9253 1149 : += record_stmt_cost (cost_vec, cnunits, scalar_store,
9254 : slp_node, 0, vect_body);
9255 1149 : continue;
9256 1149 : }
9257 :
9258 379 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
9259 379 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
9260 379 : unsigned HOST_WIDE_INT const_offset_nunits
9261 379 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
9262 379 : vec<constructor_elt, va_gc> *ctor_elts;
9263 379 : vec_alloc (ctor_elts, const_nunits);
9264 379 : gimple_seq stmts = NULL;
9265 379 : tree elt_type = TREE_TYPE (vectype);
9266 379 : unsigned HOST_WIDE_INT elt_size
9267 379 : = tree_to_uhwi (TYPE_SIZE (elt_type));
9268 : /* We support offset vectors with more elements
9269 : than the data vector for now. */
9270 379 : unsigned HOST_WIDE_INT factor
9271 : = const_offset_nunits / const_nunits;
9272 379 : vec_offset = vec_offsets[j / factor];
9273 379 : unsigned elt_offset
9274 379 : = (j % factor) * const_nunits;
9275 379 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
9276 379 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
9277 379 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
9278 1922 : for (unsigned k = 0; k < const_nunits; ++k)
9279 : {
9280 : /* Compute the offsetted pointer. */
9281 1164 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
9282 : bitsize_int (k + elt_offset));
9283 1164 : tree idx
9284 2328 : = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
9285 1164 : vec_offset, TYPE_SIZE (idx_type), boff);
9286 1164 : idx = gimple_convert (&stmts, sizetype, idx);
9287 1164 : idx = gimple_build (&stmts, MULT_EXPR, sizetype,
9288 : idx, scale);
9289 1164 : tree ptr
9290 1164 : = gimple_build (&stmts, PLUS_EXPR,
9291 1164 : TREE_TYPE (dataref_ptr),
9292 : dataref_ptr, idx);
9293 1164 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
9294 : /* Extract the element to be stored. */
9295 1164 : tree elt
9296 2328 : = gimple_build (&stmts, BIT_FIELD_REF,
9297 1164 : TREE_TYPE (vectype),
9298 1164 : vec_oprnd, TYPE_SIZE (elt_type),
9299 1164 : bitsize_int (k * elt_size));
9300 1164 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
9301 1164 : stmts = NULL;
9302 1164 : tree ref
9303 1164 : = build2 (MEM_REF, ltype, ptr,
9304 : build_int_cst (ref_type, 0));
9305 1164 : new_stmt = gimple_build_assign (ref, elt);
9306 1164 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9307 : }
9308 :
9309 379 : slp_node->push_vec_def (new_stmt);
9310 : }
9311 : }
9312 :
9313 1460 : if (costing_p)
9314 : {
9315 985 : if (dump_enabled_p ())
9316 78 : dump_printf_loc (MSG_NOTE, vect_location,
9317 : "vect_model_store_cost: inside_cost = %d, "
9318 : "prologue_cost = %d .\n",
9319 : inside_cost, prologue_cost);
9320 985 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9321 985 : slp_node->data = new vect_load_store_data (std::move (ls));
9322 : }
9323 :
9324 1460 : return true;
9325 1460 : }
9326 :
9327 1355906 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
9328 : || memory_access_type == VMAT_CONTIGUOUS_DOWN
9329 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
9330 :
9331 1355906 : unsigned inside_cost = 0, prologue_cost = 0;
9332 : /* For costing some adjacent vector stores, we'd like to cost with
9333 : the total number of them once instead of cost each one by one. */
9334 1355906 : unsigned int n_adjacent_stores = 0;
9335 1355906 : auto_vec<tree> result_chain (group_size);
9336 1355906 : auto_vec<tree, 1> vec_oprnds;
9337 1355906 : gimple *new_stmt;
9338 1355906 : if (!costing_p)
9339 : {
9340 : /* Get vectorized arguments for SLP_NODE. */
9341 553051 : vect_get_slp_defs (op_node, &vec_oprnds);
9342 553051 : vec_oprnd = vec_oprnds[0];
9343 553051 : if (mask_node)
9344 : {
9345 457 : vect_get_slp_defs (mask_node, &vec_masks);
9346 457 : vec_mask = vec_masks[0];
9347 : }
9348 : }
9349 :
9350 : /* We should have caught mismatched types earlier. */
9351 553051 : gcc_assert (costing_p
9352 : || useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
9353 1355906 : bool simd_lane_access_p
9354 1355906 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
9355 1355906 : if (!costing_p
9356 1355906 : && simd_lane_access_p
9357 4358 : && !loop_masks
9358 4358 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
9359 4358 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
9360 4358 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
9361 4358 : && integer_zerop (DR_INIT (first_dr_info->dr))
9362 1360264 : && alias_sets_conflict_p (get_alias_set (aggr_type),
9363 4358 : get_alias_set (TREE_TYPE (ref_type))))
9364 : {
9365 4350 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
9366 4350 : dataref_offset = build_int_cst (ref_type, 0);
9367 : }
9368 1351556 : else if (!costing_p)
9369 1097394 : dataref_ptr = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
9370 : simd_lane_access_p ? loop : NULL,
9371 : offset, &dummy, gsi, NULL,
9372 : simd_lane_access_p, dr_increment);
9373 :
9374 1355906 : new_stmt = NULL;
9375 1355906 : gcc_assert (!grouped_store);
9376 3015620 : for (i = 0; i < vec_num; i++)
9377 : {
9378 1659714 : if (!costing_p)
9379 685580 : vec_oprnd = vec_oprnds[i];
9380 :
9381 1659714 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
9382 : {
9383 3335 : if (costing_p)
9384 2197 : inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
9385 : slp_node, 0, vect_body);
9386 : else
9387 : {
9388 1138 : tree perm_mask = perm_mask_for_reverse (vectype);
9389 1138 : tree new_temp = make_ssa_name (vectype);
9390 :
9391 : /* Generate the permute statement. */
9392 1138 : gimple *perm_stmt
9393 1138 : = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
9394 : vec_oprnd, perm_mask);
9395 1138 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9396 :
9397 1138 : perm_stmt = SSA_NAME_DEF_STMT (new_temp);
9398 1659714 : vec_oprnd = new_temp;
9399 : }
9400 : }
9401 :
9402 1659714 : if (costing_p)
9403 : {
9404 974134 : n_adjacent_stores++;
9405 974134 : continue;
9406 : }
9407 :
9408 685580 : tree final_mask = NULL_TREE;
9409 685580 : tree final_len = NULL_TREE;
9410 685580 : tree bias = NULL_TREE;
9411 685580 : if (loop_masks)
9412 78 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
9413 : vec_num, vectype, i);
9414 685580 : if (vec_mask)
9415 658 : vec_mask = vec_masks[i];
9416 658 : if (vec_mask)
9417 658 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
9418 : vec_mask, gsi);
9419 :
9420 685580 : if (i > 0)
9421 : /* Bump the vector pointer. */
9422 132529 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
9423 : dr_bump);
9424 :
9425 685580 : unsigned misalign;
9426 685580 : unsigned HOST_WIDE_INT align;
9427 685580 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
9428 685580 : if (alignment_support_scheme == dr_aligned)
9429 : misalign = 0;
9430 314066 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
9431 : {
9432 163633 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
9433 163633 : misalign = 0;
9434 : }
9435 : else
9436 150433 : misalign = misalignment;
9437 685580 : if (dataref_offset == NULL_TREE
9438 680221 : && TREE_CODE (dataref_ptr) == SSA_NAME)
9439 185844 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, misalign);
9440 685580 : align = least_bit_hwi (misalign | align);
9441 :
9442 : /* Compute IFN when LOOP_LENS or final_mask valid. */
9443 685580 : machine_mode vmode = TYPE_MODE (vectype);
9444 685580 : machine_mode new_vmode = vmode;
9445 685580 : internal_fn partial_ifn = IFN_LAST;
9446 685580 : if (loop_lens)
9447 : {
9448 0 : opt_machine_mode new_ovmode
9449 0 : = get_len_load_store_mode (vmode, false, &partial_ifn);
9450 0 : new_vmode = new_ovmode.require ();
9451 0 : unsigned factor
9452 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
9453 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
9454 : vec_num, vectype, i, factor, true);
9455 : }
9456 685580 : else if (final_mask)
9457 : {
9458 671 : if (!can_vec_mask_load_store_p (vmode,
9459 671 : TYPE_MODE (TREE_TYPE (final_mask)),
9460 : false, &partial_ifn))
9461 0 : gcc_unreachable ();
9462 : }
9463 :
9464 685580 : if (partial_ifn == IFN_MASK_LEN_STORE)
9465 : {
9466 0 : if (!final_len)
9467 : {
9468 : /* Pass VF value to 'len' argument of
9469 : MASK_LEN_STORE if LOOP_LENS is invalid. */
9470 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
9471 : }
9472 0 : if (!final_mask)
9473 : {
9474 : /* Pass all ones value to 'mask' argument of
9475 : MASK_LEN_STORE if final_mask is invalid. */
9476 0 : mask_vectype = truth_type_for (vectype);
9477 0 : final_mask = build_minus_one_cst (mask_vectype);
9478 : }
9479 : }
9480 685580 : if (final_len)
9481 : {
9482 0 : signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
9483 0 : bias = build_int_cst (intQI_type_node, biasval);
9484 : }
9485 :
9486 : /* Arguments are ready. Create the new vector stmt. */
9487 0 : if (final_len)
9488 : {
9489 0 : gcall *call;
9490 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9491 : /* Need conversion if it's wrapped with VnQI. */
9492 0 : if (vmode != new_vmode)
9493 : {
9494 0 : tree new_vtype
9495 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
9496 : new_vmode);
9497 0 : tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
9498 0 : vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
9499 0 : gassign *new_stmt
9500 0 : = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
9501 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9502 0 : vec_oprnd = var;
9503 : }
9504 :
9505 0 : if (partial_ifn == IFN_MASK_LEN_STORE)
9506 0 : call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
9507 : dataref_ptr, ptr, final_mask,
9508 : final_len, bias, vec_oprnd);
9509 : else
9510 0 : call = gimple_build_call_internal (IFN_LEN_STORE, 5,
9511 : dataref_ptr, ptr, final_len,
9512 : bias, vec_oprnd);
9513 0 : gimple_call_set_nothrow (call, true);
9514 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9515 0 : new_stmt = call;
9516 : }
9517 685580 : else if (final_mask)
9518 : {
9519 671 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
9520 671 : gcall *call
9521 671 : = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
9522 : ptr, final_mask, vec_oprnd);
9523 671 : gimple_call_set_nothrow (call, true);
9524 671 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
9525 671 : new_stmt = call;
9526 : }
9527 : else
9528 : {
9529 684909 : data_ref = fold_build2 (MEM_REF, vectype, dataref_ptr,
9530 : dataref_offset ? dataref_offset
9531 : : build_int_cst (ref_type, 0));
9532 684909 : if (alignment_support_scheme == dr_aligned
9533 684909 : && align >= TYPE_ALIGN_UNIT (vectype))
9534 : ;
9535 : else
9536 313553 : TREE_TYPE (data_ref)
9537 627106 : = build_aligned_type (TREE_TYPE (data_ref),
9538 : align * BITS_PER_UNIT);
9539 684909 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
9540 684909 : new_stmt = gimple_build_assign (data_ref, vec_oprnd);
9541 684909 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
9542 : }
9543 : }
9544 :
9545 1355906 : if (costing_p)
9546 : {
9547 802855 : if (n_adjacent_stores > 0)
9548 802855 : vect_get_store_cost (vinfo, stmt_info, slp_node, n_adjacent_stores,
9549 : alignment_support_scheme, misalignment,
9550 : &inside_cost, cost_vec);
9551 :
9552 : /* When vectorizing a store into the function result assign
9553 : a penalty if the function returns in a multi-register location.
9554 : In this case we assume we'll end up with having to spill the
9555 : vector result and do piecewise loads as a conservative estimate. */
9556 802855 : tree base = get_base_address (STMT_VINFO_DATA_REF (stmt_info)->ref);
9557 802855 : if (base
9558 802855 : && (TREE_CODE (base) == RESULT_DECL
9559 752502 : || (DECL_P (base) && cfun_returns (base)))
9560 865044 : && !aggregate_value_p (base, cfun->decl))
9561 : {
9562 11144 : rtx reg = hard_function_value (TREE_TYPE (base), cfun->decl, 0, 1);
9563 : /* ??? Handle PARALLEL in some way. */
9564 11144 : if (REG_P (reg))
9565 : {
9566 10936 : int nregs = hard_regno_nregs (REGNO (reg), GET_MODE (reg));
9567 : /* Assume that a single reg-reg move is possible and cheap,
9568 : do not account for vector to gp register move cost. */
9569 10936 : if (nregs > 1)
9570 : {
9571 : /* Spill. */
9572 10072 : prologue_cost
9573 10072 : += record_stmt_cost (cost_vec, 1, vector_store,
9574 : slp_node, 0, vect_epilogue);
9575 : /* Loads. */
9576 10072 : prologue_cost
9577 10072 : += record_stmt_cost (cost_vec, nregs, scalar_load,
9578 : slp_node, 0, vect_epilogue);
9579 : }
9580 : }
9581 : }
9582 802855 : if (dump_enabled_p ())
9583 14051 : dump_printf_loc (MSG_NOTE, vect_location,
9584 : "vect_model_store_cost: inside_cost = %d, "
9585 : "prologue_cost = %d .\n",
9586 : inside_cost, prologue_cost);
9587 :
9588 802855 : SLP_TREE_TYPE (slp_node) = store_vec_info_type;
9589 802855 : slp_node->data = new vect_load_store_data (std::move (ls));
9590 : }
9591 :
9592 1355906 : return true;
9593 1388572 : }
9594 :
9595 : /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9596 : VECTOR_CST mask. No checks are made that the target platform supports the
9597 : mask, so callers may wish to test can_vec_perm_const_p separately, or use
9598 : vect_gen_perm_mask_checked. */
9599 :
9600 : tree
9601 63184 : vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
9602 : {
9603 63184 : tree mask_type;
9604 :
9605 63184 : poly_uint64 nunits = sel.length ();
9606 63184 : gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
9607 :
9608 63184 : mask_type = build_vector_type (ssizetype, nunits);
9609 63184 : return vec_perm_indices_to_tree (mask_type, sel);
9610 : }
9611 :
9612 : /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9613 : i.e. that the target supports the pattern _for arbitrary input vectors_. */
9614 :
9615 : tree
9616 60320 : vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
9617 : {
9618 60320 : machine_mode vmode = TYPE_MODE (vectype);
9619 60320 : gcc_assert (can_vec_perm_const_p (vmode, vmode, sel));
9620 60320 : return vect_gen_perm_mask_any (vectype, sel);
9621 : }
9622 :
9623 : /* Given a vector variable X and Y, that was generated for the scalar
9624 : STMT_INFO, generate instructions to permute the vector elements of X and Y
9625 : using permutation mask MASK_VEC, insert them at *GSI and return the
9626 : permuted vector variable. */
9627 :
9628 : static tree
9629 1442 : permute_vec_elements (vec_info *vinfo,
9630 : tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
9631 : gimple_stmt_iterator *gsi)
9632 : {
9633 1442 : tree vectype = TREE_TYPE (x);
9634 1442 : tree perm_dest, data_ref;
9635 1442 : gimple *perm_stmt;
9636 :
9637 1442 : tree scalar_dest = gimple_get_lhs (stmt_info->stmt);
9638 1442 : if (scalar_dest && TREE_CODE (scalar_dest) == SSA_NAME)
9639 1442 : perm_dest = vect_create_destination_var (scalar_dest, vectype);
9640 : else
9641 0 : perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
9642 1442 : data_ref = make_ssa_name (perm_dest);
9643 :
9644 : /* Generate the permute statement. */
9645 1442 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
9646 1442 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
9647 :
9648 1442 : return data_ref;
9649 : }
9650 :
9651 : /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9652 : inserting them on the loops preheader edge. Returns true if we
9653 : were successful in doing so (and thus STMT_INFO can be moved then),
9654 : otherwise returns false. HOIST_P indicates if we want to hoist the
9655 : definitions of all SSA uses, it would be false when we are costing. */
9656 :
9657 : static bool
9658 3974 : hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
9659 : {
9660 3974 : ssa_op_iter i;
9661 3974 : use_operand_p use_p;
9662 3974 : auto_vec<use_operand_p, 8> to_hoist;
9663 :
9664 7524 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, i, SSA_OP_USE)
9665 : {
9666 3580 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9667 3580 : if (!gimple_nop_p (def_stmt)
9668 3580 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
9669 : {
9670 : /* Make sure we don't need to recurse. While we could do
9671 : so in simple cases when there are more complex use webs
9672 : we don't have an easy way to preserve stmt order to fulfil
9673 : dependencies within them. */
9674 87 : tree op2;
9675 87 : ssa_op_iter i2;
9676 87 : if (gimple_code (def_stmt) == GIMPLE_PHI
9677 87 : || (single_ssa_def_operand (def_stmt, SSA_OP_DEF)
9678 : == NULL_DEF_OPERAND_P))
9679 30 : return false;
9680 176 : FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
9681 : {
9682 119 : gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
9683 119 : if (!gimple_nop_p (def_stmt2)
9684 119 : && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
9685 : return false;
9686 : }
9687 57 : to_hoist.safe_push (use_p);
9688 : }
9689 : }
9690 :
9691 7918 : if (to_hoist.is_empty ())
9692 : return true;
9693 :
9694 33 : if (!hoist_p)
9695 : return true;
9696 :
9697 : /* Instead of moving defs we copy them so we can zero their UID to not
9698 : confuse dominance queries in the preheader. */
9699 3 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
9700 12 : for (use_operand_p use_p : to_hoist)
9701 : {
9702 3 : gimple *def_stmt = SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p));
9703 3 : gimple *copy = gimple_copy (def_stmt);
9704 3 : gimple_set_uid (copy, 0);
9705 3 : def_operand_p def_p = single_ssa_def_operand (def_stmt, SSA_OP_DEF);
9706 3 : tree new_def = duplicate_ssa_name (DEF_FROM_PTR (def_p), copy);
9707 3 : update_stmt (copy);
9708 3 : def_p = single_ssa_def_operand (copy, SSA_OP_DEF);
9709 3 : SET_DEF (def_p, new_def);
9710 3 : SET_USE (use_p, new_def);
9711 3 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
9712 : }
9713 :
9714 : return true;
9715 3974 : }
9716 :
9717 : /* vectorizable_load.
9718 :
9719 : Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9720 : that can be vectorized.
9721 : If COST_VEC is passed, calculate costs but don't change anything,
9722 : otherwise, vectorize STMT_INFO: create a vectorized stmt to replace
9723 : it, and insert it at GSI.
9724 : Return true if STMT_INFO is vectorizable in this way. */
9725 :
9726 : static bool
9727 2223794 : vectorizable_load (vec_info *vinfo,
9728 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
9729 : slp_tree slp_node,
9730 : stmt_vector_for_cost *cost_vec)
9731 : {
9732 2223794 : tree scalar_dest;
9733 2223794 : tree vec_dest = NULL;
9734 2223794 : tree data_ref = NULL;
9735 2223794 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
9736 2223794 : class loop *loop = NULL;
9737 2223794 : class loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
9738 2223794 : bool nested_in_vect_loop = false;
9739 2223794 : tree elem_type;
9740 : /* Avoid false positive uninitialized warning, see PR110652. */
9741 2223794 : tree new_temp = NULL_TREE;
9742 2223794 : machine_mode mode;
9743 2223794 : tree dummy;
9744 2223794 : tree dataref_ptr = NULL_TREE;
9745 2223794 : tree dataref_offset = NULL_TREE;
9746 2223794 : int i, j;
9747 2223794 : unsigned int group_size;
9748 2223794 : poly_uint64 group_gap_adj;
9749 2223794 : tree msq = NULL_TREE, lsq;
9750 2223794 : tree realignment_token = NULL_TREE;
9751 2223794 : gphi *phi = NULL;
9752 2223794 : bool grouped_load = false;
9753 2223794 : stmt_vec_info first_stmt_info;
9754 2223794 : stmt_vec_info first_stmt_info_for_drptr = NULL;
9755 2223794 : bool compute_in_loop = false;
9756 2223794 : class loop *at_loop;
9757 2223794 : int vec_num;
9758 2223794 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
9759 2223794 : poly_uint64 vf;
9760 2223794 : tree aggr_type;
9761 2223794 : tree ref_type;
9762 2223794 : enum vect_def_type mask_dt = vect_unknown_def_type;
9763 2223794 : enum vect_def_type els_dt = vect_unknown_def_type;
9764 :
9765 2223794 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
9766 : return false;
9767 :
9768 2223794 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
9769 244891 : && cost_vec)
9770 : return false;
9771 :
9772 1978903 : if (!STMT_VINFO_DATA_REF (stmt_info))
9773 : return false;
9774 :
9775 1576544 : tree mask_vectype = NULL_TREE;
9776 1576544 : tree els = NULL_TREE; tree els_vectype = NULL_TREE;
9777 :
9778 1576544 : int mask_index = -1;
9779 1576544 : int els_index = -1;
9780 1576544 : slp_tree mask_node = NULL;
9781 1576544 : slp_tree els_op = NULL;
9782 1576544 : if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
9783 : {
9784 1572434 : scalar_dest = gimple_assign_lhs (assign);
9785 1572434 : if (TREE_CODE (scalar_dest) != SSA_NAME)
9786 : return false;
9787 :
9788 742561 : tree_code code = gimple_assign_rhs_code (assign);
9789 742561 : if (code != ARRAY_REF
9790 742561 : && code != BIT_FIELD_REF
9791 742561 : && code != INDIRECT_REF
9792 517341 : && code != COMPONENT_REF
9793 517341 : && code != IMAGPART_EXPR
9794 367711 : && code != REALPART_EXPR
9795 367711 : && code != MEM_REF
9796 284 : && TREE_CODE_CLASS (code) != tcc_declaration)
9797 : return false;
9798 : }
9799 : else
9800 : {
9801 4110 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
9802 4110 : if (!call || !gimple_call_internal_p (call))
9803 : return false;
9804 :
9805 4110 : internal_fn ifn = gimple_call_internal_fn (call);
9806 4110 : if (!internal_load_fn_p (ifn))
9807 : return false;
9808 :
9809 2780 : scalar_dest = gimple_call_lhs (call);
9810 2780 : if (!scalar_dest)
9811 : return false;
9812 :
9813 2780 : mask_index = internal_fn_mask_index (ifn);
9814 2780 : if (mask_index >= 0)
9815 2780 : mask_index = vect_slp_child_index_for_operand (stmt_info, mask_index);
9816 2780 : if (mask_index >= 0
9817 2780 : && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
9818 : &mask_node, &mask_dt, &mask_vectype))
9819 : return false;
9820 :
9821 2780 : els_index = internal_fn_else_index (ifn);
9822 2780 : if (els_index >= 0)
9823 2780 : els_index = vect_slp_child_index_for_operand (stmt_info, els_index);
9824 2780 : if (els_index >= 0
9825 2780 : && !vect_is_simple_use (vinfo, slp_node, els_index,
9826 : &els, &els_op, &els_dt, &els_vectype))
9827 : return false;
9828 : }
9829 :
9830 745274 : tree vectype = SLP_TREE_VECTYPE (slp_node);
9831 745274 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
9832 :
9833 745274 : if (loop_vinfo)
9834 : {
9835 513031 : loop = LOOP_VINFO_LOOP (loop_vinfo);
9836 513031 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
9837 513031 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9838 : }
9839 : else
9840 : vf = 1;
9841 :
9842 745274 : vec_num = vect_get_num_copies (vinfo, slp_node);
9843 :
9844 : /* FORNOW. This restriction should be relaxed. */
9845 745274 : if (nested_in_vect_loop && vec_num > 1)
9846 : {
9847 319 : if (dump_enabled_p ())
9848 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9849 : "multiple types in nested loop.\n");
9850 : return false;
9851 : }
9852 :
9853 744955 : elem_type = TREE_TYPE (vectype);
9854 744955 : mode = TYPE_MODE (vectype);
9855 :
9856 : /* FORNOW. In some cases can vectorize even if data-type not supported
9857 : (e.g. - data copies). */
9858 744955 : if (!can_implement_p (mov_optab, mode))
9859 : {
9860 0 : if (dump_enabled_p ())
9861 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9862 : "Aligned load, but unsupported type.\n");
9863 : return false;
9864 : }
9865 :
9866 : /* Check if the load is a part of an interleaving chain. */
9867 744955 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
9868 : {
9869 327457 : grouped_load = true;
9870 : /* FORNOW */
9871 327457 : gcc_assert (!nested_in_vect_loop);
9872 327457 : gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
9873 :
9874 327457 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
9875 327457 : group_size = DR_GROUP_SIZE (first_stmt_info);
9876 :
9877 : /* Invalidate assumptions made by dependence analysis when vectorization
9878 : on the unrolled body effectively re-orders stmts. */
9879 327457 : if (STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
9880 327457 : && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
9881 : STMT_VINFO_MIN_NEG_DIST (stmt_info)))
9882 : {
9883 0 : if (dump_enabled_p ())
9884 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9885 : "cannot perform implicit CSE when performing "
9886 : "group loads with negative dependence distance\n");
9887 : return false;
9888 : }
9889 : }
9890 : else
9891 : group_size = 1;
9892 :
9893 744955 : vect_load_store_data _ls_data{};
9894 744955 : vect_load_store_data &ls = slp_node->get_data (_ls_data);
9895 744955 : if (cost_vec
9896 744955 : && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
9897 : VLS_LOAD, &ls))
9898 : return false;
9899 : /* Temporary aliases to analysis data, should not be modified through
9900 : these. */
9901 631118 : const vect_memory_access_type memory_access_type = ls.memory_access_type;
9902 631118 : const dr_alignment_support alignment_support_scheme
9903 : = ls.alignment_support_scheme;
9904 631118 : const int misalignment = ls.misalignment;
9905 631118 : const poly_int64 poffset = ls.poffset;
9906 631118 : const vec<int> &elsvals = ls.elsvals;
9907 :
9908 631118 : int maskload_elsval = 0;
9909 631118 : bool need_zeroing = false;
9910 :
9911 : /* We might need to explicitly zero inactive elements if there are
9912 : padding bits in the type that might leak otherwise.
9913 : Refer to PR115336. */
9914 631118 : tree scalar_type = TREE_TYPE (scalar_dest);
9915 631118 : bool type_mode_padding_p
9916 1262236 : = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
9917 :
9918 631118 : if (slp_node->ldst_lanes
9919 0 : && memory_access_type != VMAT_LOAD_STORE_LANES)
9920 : {
9921 0 : if (dump_enabled_p ())
9922 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9923 : "discovered load-lane but cannot use it.\n");
9924 : return false;
9925 : }
9926 :
9927 631118 : if (mask_node)
9928 : {
9929 2654 : if (memory_access_type == VMAT_CONTIGUOUS)
9930 : {
9931 2079 : machine_mode vec_mode = TYPE_MODE (vectype);
9932 720 : if (!VECTOR_MODE_P (vec_mode)
9933 4158 : || !can_vec_mask_load_store_p (vec_mode,
9934 2079 : TYPE_MODE (mask_vectype),
9935 : true, NULL, &ls.elsvals))
9936 : return false;
9937 : }
9938 575 : else if (memory_access_type == VMAT_ELEMENTWISE
9939 575 : || memory_access_type == VMAT_STRIDED_SLP)
9940 : {
9941 0 : if (dump_enabled_p ())
9942 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9943 : "unsupported masked strided access.\n");
9944 : return false;
9945 : }
9946 575 : else if (memory_access_type != VMAT_LOAD_STORE_LANES
9947 575 : && !mat_gather_scatter_p (memory_access_type))
9948 : {
9949 32 : if (dump_enabled_p ())
9950 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9951 : "unsupported access type for masked load.\n");
9952 : return false;
9953 : }
9954 543 : else if (memory_access_type == VMAT_GATHER_SCATTER_EMULATED)
9955 : {
9956 212 : if (dump_enabled_p ())
9957 28 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9958 : "unsupported masked emulated gather.\n");
9959 : return false;
9960 : }
9961 : }
9962 :
9963 630523 : bool costing_p = cost_vec;
9964 :
9965 630523 : if (costing_p) /* transformation not required. */
9966 : {
9967 458855 : if (loop_vinfo
9968 325070 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
9969 220022 : check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
9970 : VLS_LOAD, group_size, &ls,
9971 : mask_node, &ls.elsvals);
9972 :
9973 : /* If the type needs padding we must zero inactive elements.
9974 : Check if we can do that with a VEC_COND_EXPR and store the
9975 : elsval we choose in MASKLOAD_ELSVAL. */
9976 458855 : if (ls.elsvals.length ()
9977 61156 : && type_mode_padding_p
9978 7 : && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
9979 61156 : && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
9980 : {
9981 0 : if (dump_enabled_p ())
9982 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9983 : "cannot zero inactive elements.\n");
9984 : return false;
9985 : }
9986 :
9987 458855 : if (mask_node
9988 458855 : && !vect_maybe_update_slp_op_vectype (mask_node,
9989 : mask_vectype))
9990 : {
9991 0 : if (dump_enabled_p ())
9992 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9993 : "incompatible vector types for invariants\n");
9994 : return false;
9995 : }
9996 :
9997 458855 : if (dump_enabled_p ()
9998 26066 : && memory_access_type != VMAT_ELEMENTWISE
9999 25955 : && !mat_gather_scatter_p (memory_access_type)
10000 25640 : && memory_access_type != VMAT_STRIDED_SLP
10001 25640 : && memory_access_type != VMAT_INVARIANT
10002 483531 : && alignment_support_scheme != dr_aligned)
10003 10225 : dump_printf_loc (MSG_NOTE, vect_location,
10004 : "Vectorizing an unaligned access.\n");
10005 :
10006 458855 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10007 0 : vinfo->any_known_not_updated_vssa = true;
10008 : }
10009 :
10010 : /* For now just use the first available else value.
10011 : get_supported_else_vals tries MASK_LOAD_ELSE_ZERO first so we will
10012 : select it here if it is supported. */
10013 630523 : if (elsvals.length ())
10014 84440 : maskload_elsval = *elsvals.begin ();
10015 :
10016 630523 : if (dump_enabled_p () && !costing_p)
10017 16881 : dump_printf_loc (MSG_NOTE, vect_location, "transform load.\n");
10018 :
10019 : /* Transform. */
10020 :
10021 630523 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info), *first_dr_info = NULL;
10022 630523 : if (!costing_p)
10023 171668 : ensure_base_align (dr_info);
10024 :
10025 630523 : if (memory_access_type == VMAT_INVARIANT)
10026 : {
10027 4214 : gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
10028 : /* If we have versioned for aliasing or the loop doesn't
10029 : have any data dependencies that would preclude this,
10030 : then we are sure this is a loop invariant load and
10031 : thus we can insert it on the preheader edge.
10032 : TODO: hoist_defs_of_uses should ideally be computed
10033 : once at analysis time, remembered and used in the
10034 : transform time. */
10035 8428 : bool hoist_p = (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
10036 4214 : && !nested_in_vect_loop);
10037 :
10038 4214 : bool uniform_p = true;
10039 17662 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10040 : {
10041 : /* It is unsafe to hoist a conditional load over the conditions that
10042 : make it valid. When early break this means that any invariant load
10043 : can't be hoisted unless it's in the loop header or if we know
10044 : something else has verified the load is valid to do. Alignment
10045 : peeling would do this since getting through the prologue means the
10046 : load was done at least once and so the vector main body is free to
10047 : hoist it. However today GCC will hoist the load above the PFA
10048 : loop. As such that makes it still invalid and so we can't allow it
10049 : today. */
10050 5020 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
10051 1224 : && !DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (sinfo))
10052 6208 : && gimple_bb (STMT_VINFO_STMT (vect_orig_stmt (sinfo)))
10053 1188 : != loop->header)
10054 : {
10055 1090 : if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
10056 1090 : && dump_enabled_p ())
10057 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10058 : "not hoisting invariant load due to early break"
10059 : "constraints\n");
10060 1084 : else if (dump_enabled_p ())
10061 26 : dump_printf_loc (MSG_NOTE, vect_location,
10062 : "not hoisting invariant load due to early break"
10063 : "constraints\n");
10064 : hoist_p = false;
10065 : }
10066 :
10067 3962 : hoist_p = hoist_p && hoist_defs_of_uses (sinfo->stmt, loop, false);
10068 5020 : if (sinfo != SLP_TREE_SCALAR_STMTS (slp_node)[0])
10069 303 : uniform_p = false;
10070 : }
10071 4214 : if (costing_p)
10072 : {
10073 3386 : if (!uniform_p && (!hoist_p || !vf.is_constant ()))
10074 : {
10075 0 : if (dump_enabled_p ())
10076 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10077 : "not vectorizing non-uniform invariant "
10078 : "load\n");
10079 : return false;
10080 : }
10081 1600 : enum vect_cost_model_location cost_loc
10082 3386 : = hoist_p ? vect_prologue : vect_body;
10083 3386 : unsigned int cost = record_stmt_cost (cost_vec, 1, scalar_load,
10084 : slp_node, 0, cost_loc);
10085 3386 : cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
10086 : slp_node, 0, cost_loc);
10087 3386 : unsigned int prologue_cost = hoist_p ? cost : 0;
10088 1600 : unsigned int inside_cost = hoist_p ? 0 : cost;
10089 3386 : if (dump_enabled_p ())
10090 553 : dump_printf_loc (MSG_NOTE, vect_location,
10091 : "vect_model_load_cost: inside_cost = %d, "
10092 : "prologue_cost = %d .\n",
10093 : inside_cost, prologue_cost);
10094 3386 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10095 3386 : slp_node->data = new vect_load_store_data (std::move (ls));
10096 3386 : return true;
10097 : }
10098 828 : if (hoist_p)
10099 : {
10100 : /* ??? For non-uniform lanes there could be still duplicates.
10101 : We're leaving those to post-vectorizer CSE for the moment. */
10102 626 : auto_vec<tree> scalar_defs (SLP_TREE_LANES (slp_node));
10103 2045 : for (stmt_vec_info sinfo : SLP_TREE_SCALAR_STMTS (slp_node))
10104 : {
10105 730 : gassign *stmt = as_a <gassign *> (sinfo->stmt);
10106 730 : if (dump_enabled_p ())
10107 376 : dump_printf_loc (MSG_NOTE, vect_location,
10108 : "hoisting out of the vectorized loop: %G",
10109 : (gimple *) stmt);
10110 730 : scalar_dest = copy_ssa_name (gimple_assign_lhs (stmt));
10111 730 : tree rhs = unshare_expr (gimple_assign_rhs1 (stmt));
10112 730 : edge pe = loop_preheader_edge (loop);
10113 730 : gphi *vphi = get_virtual_phi (loop->header);
10114 730 : tree vuse;
10115 730 : if (vphi)
10116 724 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
10117 : else
10118 6 : vuse = gimple_vuse (gsi_stmt (*gsi));
10119 730 : gimple *new_stmt = gimple_build_assign (scalar_dest, rhs);
10120 730 : gimple_set_vuse (new_stmt, vuse);
10121 730 : gsi_insert_on_edge_immediate (pe, new_stmt);
10122 730 : hoist_defs_of_uses (new_stmt, loop, true);
10123 730 : if (!useless_type_conversion_p (TREE_TYPE (vectype),
10124 730 : TREE_TYPE (scalar_dest)))
10125 : {
10126 10 : tree tem = make_ssa_name (TREE_TYPE (vectype));
10127 10 : new_stmt = gimple_build_assign (tem,
10128 : NOP_EXPR, scalar_dest);
10129 10 : gsi_insert_on_edge_immediate (pe, new_stmt);
10130 10 : scalar_dest = tem;
10131 : }
10132 730 : scalar_defs.quick_push (scalar_dest);
10133 730 : if (uniform_p)
10134 : break;
10135 : }
10136 626 : if (!uniform_p)
10137 : {
10138 63 : unsigned const_nunits
10139 63 : = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10140 144 : for (j = 0; j < (int) vec_num; ++j)
10141 : {
10142 81 : vec<constructor_elt, va_gc> *v = NULL;
10143 81 : vec_safe_reserve (v, const_nunits, true);
10144 522 : for (unsigned i = 0; i < const_nunits; ++i)
10145 : {
10146 360 : unsigned def_idx
10147 360 : = (j * const_nunits + i) % SLP_TREE_LANES (slp_node);
10148 360 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
10149 : scalar_defs[def_idx]);
10150 : }
10151 81 : scalar_dest = build_constructor (vectype, v);
10152 81 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10153 : vectype, NULL);
10154 81 : slp_node->push_vec_def (new_temp);
10155 : }
10156 63 : return true;
10157 : }
10158 563 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10159 : vectype, NULL);
10160 626 : }
10161 : else
10162 : {
10163 202 : gcc_assert (uniform_p);
10164 202 : gimple_stmt_iterator gsi2 = *gsi;
10165 202 : gsi_next (&gsi2);
10166 202 : new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest,
10167 : vectype, &gsi2);
10168 : }
10169 1604 : for (j = 0; j < (int) vec_num; ++j)
10170 839 : slp_node->push_vec_def (new_temp);
10171 : return true;
10172 : }
10173 :
10174 626309 : if (memory_access_type == VMAT_ELEMENTWISE
10175 626309 : || memory_access_type == VMAT_STRIDED_SLP)
10176 : {
10177 23800 : gimple_stmt_iterator incr_gsi;
10178 23800 : bool insert_after;
10179 23800 : tree offvar = NULL_TREE;
10180 23800 : tree ivstep;
10181 23800 : tree running_off;
10182 23800 : vec<constructor_elt, va_gc> *v = NULL;
10183 23800 : tree stride_base, stride_step = NULL_TREE, alias_off;
10184 : /* Checked by get_load_store_type. */
10185 23800 : unsigned int const_nunits = nunits.to_constant ();
10186 23800 : unsigned HOST_WIDE_INT cst_offset = 0;
10187 23800 : tree dr_offset;
10188 23800 : unsigned int inside_cost = 0;
10189 :
10190 23800 : gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
10191 23800 : gcc_assert (!nested_in_vect_loop);
10192 :
10193 23800 : if (grouped_load)
10194 : {
10195 : /* If we elided a consecutive load permutation, don't
10196 : use the original first statement (which could be elided)
10197 : but the one the load permutation starts with.
10198 : This ensures the stride_base below is correct. */
10199 10871 : if (!ls.subchain_p)
10200 10789 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10201 : else
10202 82 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10203 10871 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10204 10871 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10205 : }
10206 : else
10207 : {
10208 12929 : first_stmt_info = stmt_info;
10209 12929 : first_dr_info = dr_info;
10210 12929 : ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
10211 : }
10212 :
10213 23800 : if (grouped_load)
10214 : {
10215 10871 : if (memory_access_type == VMAT_STRIDED_SLP)
10216 : {
10217 : /* If we elided a consecutive load permutation, adjust
10218 : the group size here. */
10219 4297 : if (!ls.subchain_p)
10220 4215 : group_size = DR_GROUP_SIZE (first_stmt_info);
10221 : else
10222 82 : group_size = SLP_TREE_LANES (slp_node);
10223 : }
10224 : else /* VMAT_ELEMENTWISE */
10225 6574 : group_size = SLP_TREE_LANES (slp_node);
10226 : }
10227 : else
10228 : group_size = 1;
10229 :
10230 23800 : if (!costing_p)
10231 : {
10232 3487 : dr_offset = get_dr_vinfo_offset (vinfo, first_dr_info);
10233 3487 : stride_base = fold_build_pointer_plus (
10234 : DR_BASE_ADDRESS (first_dr_info->dr),
10235 : size_binop (PLUS_EXPR, convert_to_ptrofftype (dr_offset),
10236 : convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
10237 3487 : stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
10238 :
10239 : /* For a load with loop-invariant (but other than power-of-2)
10240 : stride (i.e. not a grouped access) like so:
10241 :
10242 : for (i = 0; i < n; i += stride)
10243 : ... = array[i];
10244 :
10245 : we generate a new induction variable and new accesses to
10246 : form a new vector (or vectors, depending on ncopies):
10247 :
10248 : for (j = 0; ; j += VF*stride)
10249 : tmp1 = array[j];
10250 : tmp2 = array[j + stride];
10251 : ...
10252 : vectemp = {tmp1, tmp2, ...}
10253 : */
10254 :
10255 3487 : tree increment = fold_convert (TREE_TYPE (stride_step),
10256 : LOOP_VINFO_IV_INCREMENT (loop_vinfo));
10257 3487 : ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step),
10258 : stride_step, increment);
10259 :
10260 3487 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
10261 :
10262 3487 : stride_base = cse_and_gimplify_to_preheader (loop_vinfo, stride_base);
10263 3487 : if (LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
10264 3487 : ivstep = cse_and_gimplify_to_preheader (loop_vinfo, ivstep);
10265 : else
10266 0 : ivstep = force_gimple_operand_gsi (&incr_gsi, unshare_expr (ivstep),
10267 : true, NULL_TREE, true,
10268 : GSI_SAME_STMT);
10269 3487 : create_iv (stride_base, PLUS_EXPR, ivstep, NULL, loop, &incr_gsi,
10270 : insert_after, &offvar, NULL, true);
10271 :
10272 3487 : stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step);
10273 : }
10274 :
10275 23800 : running_off = offvar;
10276 23800 : alias_off = build_int_cst (ref_type, 0);
10277 23800 : int nloads = const_nunits;
10278 23800 : int lnel = 1;
10279 23800 : tree ltype = TREE_TYPE (vectype);
10280 23800 : tree lvectype = vectype;
10281 23800 : auto_vec<tree> dr_chain;
10282 : /* ??? Modify local copies of alignment_support_scheme and
10283 : misalignment, but this part of analysis should be done
10284 : earlier and remembered, likewise the chosen load mode. */
10285 23800 : const dr_alignment_support tem = alignment_support_scheme;
10286 23800 : dr_alignment_support alignment_support_scheme = tem;
10287 23800 : const int tem2 = misalignment;
10288 23800 : int misalignment = tem2;
10289 23800 : if (memory_access_type == VMAT_STRIDED_SLP)
10290 : {
10291 17226 : HOST_WIDE_INT n = gcd (group_size, const_nunits);
10292 : /* Use the target vector type if the group size is a multiple
10293 : of it. */
10294 17226 : if (n == const_nunits)
10295 : {
10296 2272 : int mis_align = dr_misalignment (first_dr_info, vectype);
10297 : /* With VF > 1 we advance the DR by step, if that is constant
10298 : and only aligned when performed VF times, DR alignment
10299 : analysis can analyze this as aligned since it assumes
10300 : contiguous accesses. But that is not how we code generate
10301 : here, so adjust for this. */
10302 2272 : if (maybe_gt (vf, 1u)
10303 3642 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10304 3426 : DR_TARGET_ALIGNMENT (first_dr_info)))
10305 216 : mis_align = -1;
10306 2272 : dr_alignment_support dr_align
10307 2272 : = vect_supportable_dr_alignment (vinfo, dr_info, vectype,
10308 : mis_align);
10309 2272 : if (dr_align == dr_aligned
10310 2272 : || dr_align == dr_unaligned_supported)
10311 : {
10312 17226 : nloads = 1;
10313 17226 : lnel = const_nunits;
10314 17226 : ltype = vectype;
10315 17226 : alignment_support_scheme = dr_align;
10316 17226 : misalignment = mis_align;
10317 : }
10318 : }
10319 : /* Else use the biggest vector we can load the group without
10320 : accessing excess elements. */
10321 14954 : else if (n > 1)
10322 : {
10323 2014 : tree ptype;
10324 2014 : tree vtype
10325 2014 : = vector_vector_composition_type (vectype, const_nunits / n,
10326 : &ptype);
10327 2014 : if (vtype != NULL_TREE)
10328 : {
10329 1976 : dr_alignment_support dr_align;
10330 1976 : int mis_align = 0;
10331 1976 : if (VECTOR_TYPE_P (ptype))
10332 : {
10333 1056 : mis_align = dr_misalignment (first_dr_info, ptype);
10334 1056 : if (maybe_gt (vf, 1u)
10335 2082 : && !multiple_p (DR_STEP_ALIGNMENT (first_dr_info->dr),
10336 1062 : DR_TARGET_ALIGNMENT (first_dr_info)))
10337 1020 : mis_align = -1;
10338 1056 : dr_align
10339 1056 : = vect_supportable_dr_alignment (vinfo, dr_info, ptype,
10340 : mis_align);
10341 : }
10342 : else
10343 : dr_align = dr_unaligned_supported;
10344 1976 : if (dr_align == dr_aligned
10345 1976 : || dr_align == dr_unaligned_supported)
10346 : {
10347 1976 : nloads = const_nunits / n;
10348 1976 : lnel = n;
10349 1976 : lvectype = vtype;
10350 1976 : ltype = ptype;
10351 1976 : alignment_support_scheme = dr_align;
10352 1976 : misalignment = mis_align;
10353 : }
10354 : }
10355 : }
10356 17226 : unsigned align;
10357 17226 : if (alignment_support_scheme == dr_aligned)
10358 20 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
10359 : else
10360 17206 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
10361 : /* Alignment is at most the access size if we do multiple loads. */
10362 17226 : if (nloads > 1)
10363 14954 : align = MIN (tree_to_uhwi (TYPE_SIZE_UNIT (ltype)), align);
10364 17226 : ltype = build_aligned_type (ltype, align * BITS_PER_UNIT);
10365 : }
10366 :
10367 23800 : if (costing_p)
10368 : {
10369 : /* Record the composition type for target access during costing. */
10370 20313 : ls.ls_type = lvectype;
10371 20313 : ls.ls_eltype = ltype;
10372 : }
10373 : else
10374 3487 : gcc_assert (ls.ls_type == lvectype && ls.ls_eltype == ltype);
10375 :
10376 : /* For SLP permutation support we need to load the whole group,
10377 : not only the number of vector stmts the permutation result
10378 : fits in. */
10379 23800 : int ncopies;
10380 23800 : if (ls.slp_perm)
10381 : {
10382 2873 : gcc_assert (memory_access_type != VMAT_ELEMENTWISE);
10383 : /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10384 : variable VF. */
10385 2873 : unsigned int const_vf = vf.to_constant ();
10386 2873 : ncopies = CEIL (group_size * const_vf, const_nunits);
10387 2873 : dr_chain.create (ncopies);
10388 : }
10389 : else
10390 : ncopies = vec_num;
10391 :
10392 23800 : unsigned int group_el = 0;
10393 23800 : unsigned HOST_WIDE_INT
10394 23800 : elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
10395 23800 : unsigned int n_groups = 0;
10396 : /* For costing some adjacent vector loads, we'd like to cost with
10397 : the total number of them once instead of cost each one by one. */
10398 23800 : unsigned int n_adjacent_loads = 0;
10399 56862 : for (j = 0; j < ncopies; j++)
10400 : {
10401 33062 : if (nloads > 1 && !costing_p)
10402 3197 : vec_alloc (v, nloads);
10403 33062 : gimple *new_stmt = NULL;
10404 139416 : for (i = 0; i < nloads; i++)
10405 : {
10406 106354 : if (costing_p)
10407 : {
10408 : /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10409 : avoid ICE, see PR110776. */
10410 96160 : if (VECTOR_TYPE_P (ltype)
10411 5928 : && memory_access_type != VMAT_ELEMENTWISE)
10412 5928 : n_adjacent_loads++;
10413 : else
10414 90232 : inside_cost += record_stmt_cost (cost_vec, 1, scalar_load,
10415 : slp_node, 0, vect_body);
10416 96160 : continue;
10417 : }
10418 10194 : unsigned int load_el = group_el;
10419 : /* For elementwise accesses apply a load permutation directly. */
10420 10194 : if (memory_access_type == VMAT_ELEMENTWISE
10421 10194 : && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10422 2066 : load_el = SLP_TREE_LOAD_PERMUTATION (slp_node)[group_el];
10423 10194 : tree this_off = build_int_cst (TREE_TYPE (alias_off),
10424 10194 : load_el * elsz + cst_offset);
10425 10194 : tree data_ref = build2 (MEM_REF, ltype, running_off, this_off);
10426 10194 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
10427 10194 : new_temp = make_ssa_name (ltype);
10428 10194 : new_stmt = gimple_build_assign (new_temp, data_ref);
10429 10194 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
10430 10194 : if (nloads > 1)
10431 8576 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, new_temp);
10432 :
10433 10194 : group_el += lnel;
10434 10194 : if (group_el == group_size)
10435 : {
10436 9823 : n_groups++;
10437 : /* When doing SLP make sure to not load elements from
10438 : the next vector iteration, those will not be accessed
10439 : so just use the last element again. See PR107451. */
10440 9823 : if (known_lt (n_groups, vf))
10441 : {
10442 6319 : tree newoff = copy_ssa_name (running_off);
10443 6319 : gimple *incr
10444 6319 : = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
10445 : running_off, stride_step);
10446 6319 : vect_finish_stmt_generation (vinfo, stmt_info, incr, gsi);
10447 6319 : running_off = newoff;
10448 : }
10449 : group_el = 0;
10450 : }
10451 : }
10452 :
10453 33062 : if (nloads > 1)
10454 : {
10455 24292 : if (costing_p)
10456 21095 : inside_cost += record_stmt_cost (cost_vec, 1, vec_construct,
10457 : slp_node, 0, vect_body);
10458 : else
10459 : {
10460 3197 : tree vec_inv = build_constructor (lvectype, v);
10461 3197 : new_temp = vect_init_vector (vinfo, stmt_info, vec_inv,
10462 : lvectype, gsi);
10463 3197 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
10464 3197 : if (lvectype != vectype)
10465 : {
10466 398 : new_stmt
10467 398 : = gimple_build_assign (make_ssa_name (vectype),
10468 : VIEW_CONVERT_EXPR,
10469 : build1 (VIEW_CONVERT_EXPR,
10470 : vectype, new_temp));
10471 398 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10472 : gsi);
10473 : }
10474 : }
10475 : }
10476 8770 : else if (!costing_p && ltype != vectype)
10477 : {
10478 1599 : new_stmt = gimple_build_assign (make_ssa_name (vectype),
10479 : VIEW_CONVERT_EXPR,
10480 : build1 (VIEW_CONVERT_EXPR,
10481 : vectype, new_temp));
10482 1599 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
10483 : gsi);
10484 : }
10485 :
10486 33062 : if (!costing_p)
10487 : {
10488 4815 : if (ls.slp_perm)
10489 1682 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
10490 : else
10491 3133 : slp_node->push_vec_def (new_stmt);
10492 : }
10493 : }
10494 23800 : if (ls.slp_perm)
10495 : {
10496 2873 : if (costing_p)
10497 : {
10498 2080 : gcc_assert (ls.n_perms != -1U);
10499 2080 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
10500 : slp_node, 0, vect_body);
10501 : }
10502 : else
10503 : {
10504 793 : unsigned n_perms2;
10505 793 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
10506 : false, &n_perms2);
10507 793 : gcc_assert (ls.n_perms == n_perms2);
10508 : }
10509 : }
10510 :
10511 23800 : if (costing_p)
10512 : {
10513 20313 : if (n_adjacent_loads > 0)
10514 2196 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10515 : alignment_support_scheme, misalignment, false,
10516 : &inside_cost, nullptr, cost_vec, cost_vec,
10517 : true);
10518 20313 : if (dump_enabled_p ())
10519 522 : dump_printf_loc (MSG_NOTE, vect_location,
10520 : "vect_model_load_cost: inside_cost = %u, "
10521 : "prologue_cost = 0 .\n",
10522 : inside_cost);
10523 20313 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10524 20313 : slp_node->data = new vect_load_store_data (std::move (ls));
10525 : }
10526 :
10527 23800 : return true;
10528 23800 : }
10529 :
10530 602509 : if (mat_gather_scatter_p (memory_access_type)
10531 602509 : && !ls.ls_type)
10532 : grouped_load = false;
10533 :
10534 599490 : if (grouped_load
10535 602509 : || SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10536 : {
10537 279071 : if (grouped_load)
10538 : {
10539 278623 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
10540 278623 : group_size = DR_GROUP_SIZE (first_stmt_info);
10541 : }
10542 : else
10543 : {
10544 : first_stmt_info = stmt_info;
10545 : group_size = 1;
10546 : }
10547 : /* For SLP vectorization we directly vectorize a subchain
10548 : without permutation. */
10549 279071 : if (! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
10550 218044 : first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
10551 : /* For BB vectorization always use the first stmt to base
10552 : the data ref pointer on. */
10553 279071 : if (bb_vinfo)
10554 224154 : first_stmt_info_for_drptr
10555 224154 : = vect_find_first_scalar_stmt_in_slp (slp_node);
10556 :
10557 279071 : first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
10558 279071 : group_gap_adj = 0;
10559 :
10560 : /* VEC_NUM is the number of vect stmts to be created for this group. */
10561 279071 : grouped_load = false;
10562 : /* If an SLP permutation is from N elements to N elements,
10563 : and if one vector holds a whole number of N, we can load
10564 : the inputs to the permutation in the same way as an
10565 : unpermuted sequence. In other cases we need to load the
10566 : whole group, not only the number of vector stmts the
10567 : permutation result fits in. */
10568 279071 : unsigned scalar_lanes = SLP_TREE_LANES (slp_node);
10569 279071 : if (nested_in_vect_loop)
10570 : /* We do not support grouped accesses in a nested loop,
10571 : instead the access is contiguous but it might be
10572 : permuted. No gap adjustment is needed though. */
10573 : ;
10574 279069 : else if (ls.slp_perm
10575 279069 : && (group_size != scalar_lanes
10576 11633 : || !multiple_p (nunits, group_size)))
10577 : {
10578 : /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10579 : variable VF; see vect_transform_slp_perm_load. */
10580 50582 : unsigned int const_vf = vf.to_constant ();
10581 50582 : unsigned int const_nunits = nunits.to_constant ();
10582 50582 : vec_num = CEIL (group_size * const_vf, const_nunits);
10583 50582 : group_gap_adj = vf * group_size - nunits * vec_num;
10584 : }
10585 : else
10586 : {
10587 228487 : group_gap_adj = group_size - scalar_lanes;
10588 : }
10589 :
10590 279071 : ref_type = get_group_alias_ptr_type (first_stmt_info);
10591 : }
10592 : else
10593 : {
10594 323438 : first_stmt_info = stmt_info;
10595 323438 : first_dr_info = dr_info;
10596 323438 : group_size = 1;
10597 323438 : group_gap_adj = 0;
10598 323438 : ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
10599 : }
10600 :
10601 602509 : vec_loop_masks *loop_masks
10602 378355 : = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
10603 602509 : ? &LOOP_VINFO_MASKS (loop_vinfo)
10604 34 : : NULL);
10605 34 : vec_loop_lens *loop_lens
10606 378355 : = (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)
10607 : ? &LOOP_VINFO_LENS (loop_vinfo)
10608 0 : : NULL);
10609 :
10610 : /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10611 : are some difference here. We cannot enable both the lens and masks
10612 : during transform but it is allowed during analysis.
10613 : Shouldn't go with length-based approach if fully masked. */
10614 602509 : if (cost_vec == NULL)
10615 : /* The cost_vec is NULL during transform. */
10616 167353 : gcc_assert ((!loop_lens || !loop_masks));
10617 :
10618 : /* Targets with store-lane instructions must not require explicit
10619 : realignment. vect_supportable_dr_alignment always returns either
10620 : dr_aligned or dr_unaligned_supported for (non-length) masked
10621 : operations. */
10622 602509 : gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
10623 : && !mask_node
10624 : && !loop_masks)
10625 : || mat_gather_scatter_p (memory_access_type)
10626 : || alignment_support_scheme == dr_aligned
10627 : || alignment_support_scheme == dr_unaligned_supported);
10628 :
10629 : /* In case the vectorization factor (VF) is bigger than the number
10630 : of elements that we can fit in a vectype (nunits), we have to generate
10631 : more than one vector stmt - i.e - we need to "unroll" the
10632 : vector stmt by a factor VF/nunits. In doing so, we record a pointer
10633 : from one copy of the vector stmt to the next, in the field
10634 : STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10635 : stages to find the correct vector defs to be used when vectorizing
10636 : stmts that use the defs of the current stmt. The example below
10637 : illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10638 : need to create 4 vectorized stmts):
10639 :
10640 : before vectorization:
10641 : RELATED_STMT VEC_STMT
10642 : S1: x = memref - -
10643 : S2: z = x + 1 - -
10644 :
10645 : step 1: vectorize stmt S1:
10646 : We first create the vector stmt VS1_0, and, as usual, record a
10647 : pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10648 : Next, we create the vector stmt VS1_1, and record a pointer to
10649 : it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10650 : Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10651 : stmts and pointers:
10652 : RELATED_STMT VEC_STMT
10653 : VS1_0: vx0 = memref0 VS1_1 -
10654 : VS1_1: vx1 = memref1 VS1_2 -
10655 : VS1_2: vx2 = memref2 VS1_3 -
10656 : VS1_3: vx3 = memref3 - -
10657 : S1: x = load - VS1_0
10658 : S2: z = x + 1 - -
10659 : */
10660 :
10661 : /* If the data reference is aligned (dr_aligned) or potentially unaligned
10662 : on a target that supports unaligned accesses (dr_unaligned_supported)
10663 : we generate the following code:
10664 : p = initial_addr;
10665 : indx = 0;
10666 : loop {
10667 : p = p + indx * vectype_size;
10668 : vec_dest = *(p);
10669 : indx = indx + 1;
10670 : }
10671 :
10672 : Otherwise, the data reference is potentially unaligned on a target that
10673 : does not support unaligned accesses (dr_explicit_realign_optimized) -
10674 : then generate the following code, in which the data in each iteration is
10675 : obtained by two vector loads, one from the previous iteration, and one
10676 : from the current iteration:
10677 : p1 = initial_addr;
10678 : msq_init = *(floor(p1))
10679 : p2 = initial_addr + VS - 1;
10680 : realignment_token = call target_builtin;
10681 : indx = 0;
10682 : loop {
10683 : p2 = p2 + indx * vectype_size
10684 : lsq = *(floor(p2))
10685 : vec_dest = realign_load (msq, lsq, realignment_token)
10686 : indx = indx + 1;
10687 : msq = lsq;
10688 : } */
10689 :
10690 : /* If the misalignment remains the same throughout the execution of the
10691 : loop, we can create the init_addr and permutation mask at the loop
10692 : preheader. Otherwise, it needs to be created inside the loop.
10693 : This can only occur when vectorizing memory accesses in the inner-loop
10694 : nested within an outer-loop that is being vectorized. */
10695 :
10696 602509 : if (nested_in_vect_loop
10697 602509 : && !multiple_p (DR_STEP_ALIGNMENT (dr_info->dr),
10698 1234 : GET_MODE_SIZE (TYPE_MODE (vectype))))
10699 : {
10700 195 : gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
10701 : compute_in_loop = true;
10702 : }
10703 :
10704 602509 : bool diff_first_stmt_info
10705 602509 : = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
10706 :
10707 602509 : tree offset = NULL_TREE;
10708 602509 : if ((alignment_support_scheme == dr_explicit_realign_optimized
10709 602509 : || alignment_support_scheme == dr_explicit_realign)
10710 0 : && !compute_in_loop)
10711 : {
10712 : /* If we have different first_stmt_info, we can't set up realignment
10713 : here, since we can't guarantee first_stmt_info DR has been
10714 : initialized yet, use first_stmt_info_for_drptr DR by bumping the
10715 : distance from first_stmt_info DR instead as below. */
10716 0 : if (!costing_p)
10717 : {
10718 0 : if (!diff_first_stmt_info)
10719 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype, gsi,
10720 : &realignment_token,
10721 : alignment_support_scheme, NULL_TREE,
10722 : &at_loop);
10723 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
10724 : {
10725 0 : phi = as_a<gphi *> (SSA_NAME_DEF_STMT (msq));
10726 0 : offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
10727 : size_one_node);
10728 0 : gcc_assert (!first_stmt_info_for_drptr);
10729 : }
10730 : }
10731 : }
10732 : else
10733 602509 : at_loop = loop;
10734 :
10735 602509 : if (!known_eq (poffset, 0))
10736 4640 : offset = (offset
10737 4640 : ? size_binop (PLUS_EXPR, offset, size_int (poffset))
10738 4640 : : size_int (poffset));
10739 :
10740 602509 : tree dr_increment;
10741 602509 : tree dr_bump;
10742 602509 : tree vec_offset = NULL_TREE;
10743 :
10744 602509 : auto_vec<tree> vec_offsets;
10745 602509 : auto_vec<tree> vec_masks;
10746 602509 : if (mask_node && !costing_p)
10747 628 : vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
10748 : &vec_masks);
10749 :
10750 602509 : tree vec_mask = NULL_TREE;
10751 602509 : tree vec_els = NULL_TREE;
10752 602509 : if (memory_access_type == VMAT_LOAD_STORE_LANES)
10753 : {
10754 0 : const internal_fn lanes_ifn = ls.lanes_ifn;
10755 :
10756 0 : gcc_assert (alignment_support_scheme == dr_aligned
10757 : || alignment_support_scheme == dr_unaligned_supported);
10758 :
10759 0 : aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
10760 0 : if (!costing_p)
10761 : {
10762 0 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
10763 : memory_access_type);
10764 0 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
10765 : memory_access_type);
10766 : }
10767 :
10768 0 : unsigned int inside_cost = 0, prologue_cost = 0;
10769 : /* For costing some adjacent vector loads, we'd like to cost with
10770 : the total number of them once instead of cost each one by one. */
10771 0 : unsigned int n_adjacent_loads = 0;
10772 0 : int ncopies = vec_num / group_size;
10773 0 : for (j = 0; j < ncopies; j++)
10774 : {
10775 0 : if (costing_p)
10776 : {
10777 : /* An IFN_LOAD_LANES will load all its vector results,
10778 : regardless of which ones we actually need. Account
10779 : for the cost of unused results. */
10780 0 : if (first_stmt_info == stmt_info)
10781 : {
10782 0 : unsigned int gaps = DR_GROUP_SIZE (first_stmt_info);
10783 0 : stmt_vec_info next_stmt_info = first_stmt_info;
10784 0 : do
10785 : {
10786 0 : gaps -= 1;
10787 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
10788 : }
10789 0 : while (next_stmt_info);
10790 0 : if (gaps)
10791 : {
10792 0 : if (dump_enabled_p ())
10793 0 : dump_printf_loc (MSG_NOTE, vect_location,
10794 : "vect_model_load_cost: %d "
10795 : "unused vectors.\n",
10796 : gaps);
10797 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, gaps,
10798 : alignment_support_scheme,
10799 : misalignment, false, &inside_cost,
10800 : &prologue_cost, cost_vec, cost_vec,
10801 : true);
10802 : }
10803 : }
10804 0 : n_adjacent_loads++;
10805 0 : continue;
10806 0 : }
10807 :
10808 : /* 1. Create the vector or array pointer update chain. */
10809 0 : if (j == 0)
10810 0 : dataref_ptr
10811 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10812 : at_loop, offset, &dummy, gsi,
10813 : NULL, false, dr_increment);
10814 : else
10815 : {
10816 0 : gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
10817 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10818 : stmt_info, dr_bump);
10819 : }
10820 0 : if (mask_node)
10821 0 : vec_mask = vec_masks[j];
10822 :
10823 0 : tree vec_array = create_vector_array (vectype, group_size);
10824 :
10825 0 : tree final_mask = NULL_TREE;
10826 0 : tree final_len = NULL_TREE;
10827 0 : tree bias = NULL_TREE;
10828 0 : if (loop_masks)
10829 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10830 : ncopies, vectype, j);
10831 0 : if (vec_mask)
10832 0 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
10833 : vec_mask, gsi);
10834 :
10835 0 : if (lanes_ifn == IFN_MASK_LEN_LOAD_LANES)
10836 : {
10837 0 : if (loop_lens)
10838 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
10839 : ncopies, vectype, j, 1, true);
10840 : else
10841 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
10842 0 : signed char biasval
10843 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
10844 0 : bias = build_int_cst (intQI_type_node, biasval);
10845 0 : if (!final_mask)
10846 : {
10847 0 : mask_vectype = truth_type_for (vectype);
10848 0 : final_mask = build_minus_one_cst (mask_vectype);
10849 : }
10850 : }
10851 :
10852 0 : if (final_mask)
10853 : {
10854 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
10855 0 : if (type_mode_padding_p
10856 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
10857 0 : need_zeroing = true;
10858 : }
10859 :
10860 0 : gcall *call;
10861 0 : if (final_len && final_mask)
10862 : {
10863 : /* Emit:
10864 : VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10865 : VEC_MASK, LEN, BIAS). */
10866 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10867 0 : tree alias_ptr = build_int_cst (ref_type, align);
10868 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD_LANES, 6,
10869 : dataref_ptr, alias_ptr,
10870 : final_mask, vec_els,
10871 : final_len, bias);
10872 : }
10873 0 : else if (final_mask)
10874 : {
10875 : /* Emit:
10876 : VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10877 : VEC_MASK). */
10878 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (vectype));
10879 0 : tree alias_ptr = build_int_cst (ref_type, align);
10880 0 : call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 4,
10881 : dataref_ptr, alias_ptr,
10882 : final_mask, vec_els);
10883 : }
10884 : else
10885 : {
10886 : /* Emit:
10887 : VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10888 0 : data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
10889 0 : call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
10890 : }
10891 0 : gimple_call_set_lhs (call, vec_array);
10892 0 : gimple_call_set_nothrow (call, true);
10893 0 : vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
10894 :
10895 : /* Extract each vector into an SSA_NAME. */
10896 0 : for (unsigned i = 0; i < group_size; i++)
10897 : {
10898 0 : new_temp = read_vector_array (vinfo, stmt_info, gsi, scalar_dest,
10899 : vec_array, i, need_zeroing,
10900 : final_mask);
10901 0 : slp_node->push_vec_def (new_temp);
10902 : }
10903 :
10904 : /* Record that VEC_ARRAY is now dead. */
10905 0 : vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
10906 : }
10907 :
10908 0 : if (costing_p)
10909 : {
10910 0 : if (n_adjacent_loads > 0)
10911 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
10912 : alignment_support_scheme, misalignment, false,
10913 : &inside_cost, &prologue_cost, cost_vec,
10914 : cost_vec, true);
10915 0 : if (dump_enabled_p ())
10916 0 : dump_printf_loc (MSG_NOTE, vect_location,
10917 : "vect_model_load_cost: inside_cost = %u, "
10918 : "prologue_cost = %u .\n",
10919 : inside_cost, prologue_cost);
10920 0 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
10921 0 : slp_node->data = new vect_load_store_data (std::move (ls));
10922 : }
10923 :
10924 0 : return true;
10925 : }
10926 :
10927 602509 : if (mat_gather_scatter_p (memory_access_type))
10928 : {
10929 3019 : gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type);
10930 :
10931 3019 : auto_vec<tree> dr_chain (vec_num);
10932 :
10933 : /* If we pun the original vectype the loads as well as costing, length,
10934 : etc. is performed with the new type. After loading we VIEW_CONVERT
10935 : the data to the original vectype. */
10936 3019 : tree original_vectype = vectype;
10937 3019 : if (ls.ls_type)
10938 0 : vectype = ls.ls_type;
10939 :
10940 : /* 1. Create the vector or array pointer update chain. */
10941 3019 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10942 : {
10943 3019 : aggr_type = NULL_TREE;
10944 3019 : dr_increment = NULL_TREE;
10945 3019 : if (!costing_p)
10946 762 : vect_get_gather_scatter_ops (loop, slp_node, &dataref_ptr,
10947 : &vec_offsets);
10948 : }
10949 : else
10950 : {
10951 0 : aggr_type = elem_type;
10952 0 : if (!costing_p)
10953 : {
10954 0 : vect_get_strided_load_store_ops (stmt_info, slp_node, vectype,
10955 : ls.strided_offset_vectype,
10956 : loop_vinfo, gsi,
10957 : &dr_increment, &dr_bump,
10958 : &vec_offset);
10959 0 : dataref_ptr
10960 0 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
10961 : at_loop, offset, &dummy, gsi,
10962 : NULL, false, dr_increment);
10963 : }
10964 : }
10965 :
10966 3019 : unsigned int inside_cost = 0, prologue_cost = 0;
10967 :
10968 3019 : gimple *new_stmt = NULL;
10969 6799 : for (i = 0; i < vec_num; i++)
10970 : {
10971 3780 : tree final_mask = NULL_TREE;
10972 3780 : tree final_len = NULL_TREE;
10973 3780 : tree bias = NULL_TREE;
10974 3780 : if (!costing_p)
10975 : {
10976 981 : if (mask_node)
10977 156 : vec_mask = vec_masks[i];
10978 981 : if (loop_masks)
10979 0 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
10980 : vec_num, vectype, i);
10981 981 : if (vec_mask)
10982 156 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
10983 : final_mask, vec_mask, gsi);
10984 :
10985 981 : if (i > 0 && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
10986 0 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
10987 : stmt_info, dr_bump);
10988 : }
10989 :
10990 : /* 2. Create the vector-load in the loop. */
10991 3780 : unsigned align = get_object_alignment (DR_REF (first_dr_info->dr));
10992 3780 : tree alias_align_ptr = build_int_cst (ref_type, align);
10993 3780 : if (memory_access_type == VMAT_GATHER_SCATTER_IFN)
10994 : {
10995 0 : if (costing_p)
10996 : {
10997 0 : if (ls.supported_offset_vectype
10998 0 : && !tree_nop_conversion_p (ls.supported_offset_vectype,
10999 : vec_offset))
11000 0 : inside_cost
11001 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11002 : slp_node, 0, vect_body);
11003 0 : if (ls.supported_scale)
11004 0 : inside_cost
11005 0 : += record_stmt_cost (cost_vec, 1, vector_stmt,
11006 : slp_node, 0, vect_body);
11007 :
11008 0 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11009 0 : inside_cost
11010 0 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11011 : slp_node, 0, vect_body);
11012 0 : continue;
11013 0 : }
11014 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
11015 0 : vec_offset = vec_offsets[i];
11016 0 : tree zero = build_zero_cst (vectype);
11017 0 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11018 0 : bool strided = !VECTOR_TYPE_P (TREE_TYPE (vec_offset));
11019 :
11020 : /* Perform the offset conversion and scaling if necessary. */
11021 0 : if (!strided
11022 0 : && (ls.supported_offset_vectype || ls.supported_scale))
11023 : {
11024 0 : gimple_seq stmts = NULL;
11025 0 : if (ls.supported_offset_vectype)
11026 0 : vec_offset = gimple_convert
11027 0 : (&stmts, ls.supported_offset_vectype, vec_offset);
11028 0 : if (ls.supported_scale)
11029 : {
11030 : /* Only scale the vec_offset if we haven't already. */
11031 0 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
11032 0 : || i == 0)
11033 : {
11034 0 : tree mult_cst = build_int_cst
11035 0 : (TREE_TYPE (TREE_TYPE (vec_offset)),
11036 0 : SLP_TREE_GS_SCALE (slp_node) / ls.supported_scale);
11037 0 : tree mult = build_vector_from_val
11038 0 : (TREE_TYPE (vec_offset), mult_cst);
11039 0 : vec_offset = gimple_build
11040 0 : (&stmts, MULT_EXPR, TREE_TYPE (vec_offset),
11041 : vec_offset, mult);
11042 : }
11043 0 : scale = size_int (ls.supported_scale);
11044 : }
11045 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11046 : }
11047 :
11048 0 : if (ls.gs.ifn == IFN_MASK_LEN_GATHER_LOAD)
11049 : {
11050 0 : if (loop_lens)
11051 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11052 : vec_num, vectype, i, 1, true);
11053 : else
11054 0 : final_len = build_int_cst (sizetype,
11055 0 : TYPE_VECTOR_SUBPARTS (vectype));
11056 0 : signed char biasval
11057 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11058 0 : bias = build_int_cst (intQI_type_node, biasval);
11059 0 : if (!final_mask)
11060 : {
11061 0 : mask_vectype = truth_type_for (vectype);
11062 0 : final_mask = build_minus_one_cst (mask_vectype);
11063 : }
11064 : }
11065 :
11066 0 : if (final_mask)
11067 : {
11068 0 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11069 0 : if (type_mode_padding_p
11070 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11071 0 : need_zeroing = true;
11072 : }
11073 :
11074 0 : gcall *call;
11075 0 : if (final_len && final_mask)
11076 : {
11077 0 : if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
11078 0 : call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
11079 : 9, dataref_ptr,
11080 : alias_align_ptr,
11081 : vec_offset, scale, zero,
11082 : final_mask, vec_els,
11083 : final_len, bias);
11084 : else
11085 : /* Non-vector offset indicates that prefer to take
11086 : MASK_LEN_STRIDED_LOAD instead of the
11087 : MASK_LEN_GATHER_LOAD with direct stride arg. */
11088 0 : call = gimple_build_call_internal
11089 0 : (IFN_MASK_LEN_STRIDED_LOAD, 7, dataref_ptr,
11090 : vec_offset, zero, final_mask, vec_els, final_len,
11091 : bias);
11092 : }
11093 0 : else if (final_mask)
11094 0 : call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
11095 : 7, dataref_ptr,
11096 : alias_align_ptr,
11097 : vec_offset, scale,
11098 : zero, final_mask, vec_els);
11099 : else
11100 0 : call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
11101 : dataref_ptr,
11102 : alias_align_ptr,
11103 : vec_offset, scale, zero);
11104 0 : gimple_call_set_nothrow (call, true);
11105 0 : new_stmt = call;
11106 0 : data_ref = NULL_TREE;
11107 : }
11108 3780 : else if (memory_access_type == VMAT_GATHER_SCATTER_LEGACY)
11109 : {
11110 : /* The builtin decls path for gather is legacy, x86 only. */
11111 848 : gcc_assert (!final_len && nunits.is_constant ());
11112 848 : if (costing_p)
11113 : {
11114 566 : unsigned int cnunits = vect_nunits_for_cost (vectype);
11115 566 : inside_cost
11116 566 : = record_stmt_cost (cost_vec, cnunits, scalar_load,
11117 : slp_node, 0, vect_body);
11118 566 : continue;
11119 566 : }
11120 282 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11121 282 : poly_uint64 offset_nunits = TYPE_VECTOR_SUBPARTS (offset_vectype);
11122 282 : if (known_eq (nunits, offset_nunits))
11123 : {
11124 133 : new_stmt = vect_build_one_gather_load_call
11125 133 : (vinfo, stmt_info, slp_node, vectype, gsi,
11126 133 : ls.gs.decl, dataref_ptr, vec_offsets[i],
11127 : final_mask);
11128 133 : data_ref = NULL_TREE;
11129 : }
11130 149 : else if (known_eq (nunits, offset_nunits * 2))
11131 : {
11132 : /* We have a offset vector with half the number of
11133 : lanes but the builtins will produce full vectype
11134 : data with just the lower lanes filled. */
11135 63 : new_stmt = vect_build_one_gather_load_call
11136 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11137 63 : ls.gs.decl, dataref_ptr, vec_offsets[2 * i],
11138 : final_mask);
11139 63 : tree low = make_ssa_name (vectype);
11140 63 : gimple_set_lhs (new_stmt, low);
11141 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11142 :
11143 : /* now put upper half of final_mask in final_mask low. */
11144 63 : if (final_mask
11145 63 : && !SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (final_mask))))
11146 : {
11147 11 : int count = nunits.to_constant ();
11148 11 : vec_perm_builder sel (count, count, 1);
11149 11 : sel.quick_grow (count);
11150 98 : for (int i = 0; i < count; ++i)
11151 76 : sel[i] = i | (count / 2);
11152 11 : vec_perm_indices indices (sel, 2, count);
11153 11 : tree perm_mask = vect_gen_perm_mask_checked
11154 11 : (TREE_TYPE (final_mask), indices);
11155 11 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11156 : final_mask, final_mask,
11157 : perm_mask);
11158 11 : final_mask = make_ssa_name (TREE_TYPE (final_mask));
11159 11 : gimple_set_lhs (new_stmt, final_mask);
11160 11 : vect_finish_stmt_generation (vinfo, stmt_info,
11161 : new_stmt, gsi);
11162 11 : }
11163 52 : else if (final_mask)
11164 : {
11165 24 : new_stmt = gimple_build_assign (NULL_TREE,
11166 : VEC_UNPACK_HI_EXPR,
11167 : final_mask);
11168 24 : final_mask = make_ssa_name
11169 24 : (truth_type_for (offset_vectype));
11170 24 : gimple_set_lhs (new_stmt, final_mask);
11171 24 : vect_finish_stmt_generation (vinfo, stmt_info,
11172 : new_stmt, gsi);
11173 : }
11174 :
11175 63 : new_stmt = vect_build_one_gather_load_call
11176 126 : (vinfo, stmt_info, slp_node, vectype, gsi,
11177 : ls.gs.decl, dataref_ptr,
11178 63 : vec_offsets[2 * i + 1], final_mask);
11179 63 : tree high = make_ssa_name (vectype);
11180 63 : gimple_set_lhs (new_stmt, high);
11181 63 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11182 :
11183 : /* compose low + high. */
11184 63 : int count = nunits.to_constant ();
11185 63 : vec_perm_builder sel (count, count, 1);
11186 63 : sel.quick_grow (count);
11187 710 : for (int i = 0; i < count; ++i)
11188 584 : sel[i] = i < count / 2 ? i : i + count / 2;
11189 63 : vec_perm_indices indices (sel, 2, count);
11190 63 : tree perm_mask
11191 63 : = vect_gen_perm_mask_checked (vectype, indices);
11192 63 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11193 : low, high, perm_mask);
11194 63 : data_ref = NULL_TREE;
11195 63 : }
11196 86 : else if (known_eq (nunits * 2, offset_nunits))
11197 : {
11198 : /* We have a offset vector with double the number of
11199 : lanes. Select the low/high part accordingly. */
11200 86 : vec_offset = vec_offsets[i / 2];
11201 86 : if (i & 1)
11202 : {
11203 43 : int count = offset_nunits.to_constant ();
11204 43 : vec_perm_builder sel (count, count, 1);
11205 43 : sel.quick_grow (count);
11206 506 : for (int i = 0; i < count; ++i)
11207 420 : sel[i] = i | (count / 2);
11208 43 : vec_perm_indices indices (sel, 2, count);
11209 43 : tree perm_mask = vect_gen_perm_mask_checked
11210 43 : (TREE_TYPE (vec_offset), indices);
11211 43 : new_stmt = gimple_build_assign (NULL_TREE, VEC_PERM_EXPR,
11212 : vec_offset, vec_offset,
11213 : perm_mask);
11214 43 : vec_offset = make_ssa_name (TREE_TYPE (vec_offset));
11215 43 : gimple_set_lhs (new_stmt, vec_offset);
11216 43 : vect_finish_stmt_generation (vinfo, stmt_info,
11217 : new_stmt, gsi);
11218 43 : }
11219 86 : new_stmt = vect_build_one_gather_load_call
11220 86 : (vinfo, stmt_info, slp_node, vectype, gsi,
11221 : ls.gs.decl,
11222 : dataref_ptr, vec_offset, final_mask);
11223 86 : data_ref = NULL_TREE;
11224 : }
11225 : else
11226 0 : gcc_unreachable ();
11227 : }
11228 : else
11229 : {
11230 : /* Emulated gather-scatter. */
11231 2932 : gcc_assert (!final_mask);
11232 2932 : unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
11233 2932 : if (costing_p)
11234 : {
11235 : /* For emulated gathers N offset vector element
11236 : offset add is consumed by the load). */
11237 2233 : inside_cost = record_stmt_cost (cost_vec, 1, vec_deconstruct,
11238 : slp_node, 0, vect_body);
11239 : /* N scalar loads plus gathering them into a
11240 : vector. */
11241 2233 : inside_cost
11242 2233 : = record_stmt_cost (cost_vec, const_nunits, scalar_load,
11243 : slp_node, 0, vect_body);
11244 2233 : inside_cost
11245 2233 : = record_stmt_cost (cost_vec, 1, vec_construct,
11246 : slp_node, 0, vect_body);
11247 2233 : continue;
11248 : }
11249 699 : tree offset_vectype = TREE_TYPE (vec_offsets[0]);
11250 699 : unsigned HOST_WIDE_INT const_offset_nunits
11251 699 : = TYPE_VECTOR_SUBPARTS (offset_vectype).to_constant ();
11252 699 : vec<constructor_elt, va_gc> *ctor_elts;
11253 699 : vec_alloc (ctor_elts, const_nunits);
11254 699 : gimple_seq stmts = NULL;
11255 : /* We support offset vectors with more elements
11256 : than the data vector for now. */
11257 699 : unsigned HOST_WIDE_INT factor
11258 : = const_offset_nunits / const_nunits;
11259 699 : vec_offset = vec_offsets[i / factor];
11260 699 : unsigned elt_offset = (i % factor) * const_nunits;
11261 699 : tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
11262 699 : tree scale = size_int (SLP_TREE_GS_SCALE (slp_node));
11263 699 : tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
11264 3528 : for (unsigned k = 0; k < const_nunits; ++k)
11265 : {
11266 2130 : tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
11267 : bitsize_int (k + elt_offset));
11268 6390 : tree idx = gimple_build (&stmts, BIT_FIELD_REF, idx_type,
11269 2130 : vec_offset, TYPE_SIZE (idx_type),
11270 : boff);
11271 2130 : idx = gimple_convert (&stmts, sizetype, idx);
11272 2130 : idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
11273 2130 : tree ptr = gimple_build (&stmts, PLUS_EXPR,
11274 2130 : TREE_TYPE (dataref_ptr),
11275 : dataref_ptr, idx);
11276 2130 : ptr = gimple_convert (&stmts, ptr_type_node, ptr);
11277 2130 : tree elt = make_ssa_name (TREE_TYPE (vectype));
11278 2130 : tree ref = build2 (MEM_REF, ltype, ptr,
11279 : build_int_cst (ref_type, 0));
11280 2130 : new_stmt = gimple_build_assign (elt, ref);
11281 4260 : gimple_set_vuse (new_stmt, gimple_vuse (gsi_stmt (*gsi)));
11282 2130 : gimple_seq_add_stmt (&stmts, new_stmt);
11283 2130 : CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE, elt);
11284 : }
11285 699 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
11286 699 : new_stmt = gimple_build_assign (NULL_TREE,
11287 : build_constructor (vectype,
11288 : ctor_elts));
11289 699 : data_ref = NULL_TREE;
11290 : }
11291 :
11292 981 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11293 : /* DATA_REF is null if we've already built the statement. */
11294 981 : if (data_ref)
11295 : {
11296 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11297 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11298 : }
11299 1962 : new_temp = (need_zeroing
11300 981 : ? make_ssa_name (vectype)
11301 981 : : make_ssa_name (vec_dest, new_stmt));
11302 981 : gimple_set_lhs (new_stmt, new_temp);
11303 981 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11304 :
11305 : /* If we need to explicitly zero inactive elements emit a
11306 : VEC_COND_EXPR that does so. */
11307 981 : if (need_zeroing)
11308 : {
11309 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11310 : vectype);
11311 :
11312 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11313 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11314 : final_mask, new_temp, vec_els);
11315 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11316 0 : new_temp = new_temp2;
11317 : }
11318 :
11319 981 : if (ls.ls_type)
11320 : {
11321 0 : new_stmt = gimple_build_assign (make_ssa_name
11322 : (original_vectype),
11323 : VIEW_CONVERT_EXPR,
11324 : build1 (VIEW_CONVERT_EXPR,
11325 : original_vectype,
11326 : new_temp));
11327 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11328 : }
11329 :
11330 : /* Store vector loads in the corresponding SLP_NODE. */
11331 981 : if (!costing_p)
11332 : {
11333 981 : if (ls.slp_perm)
11334 0 : dr_chain.quick_push (gimple_assign_lhs (new_stmt));
11335 : else
11336 981 : slp_node->push_vec_def (new_stmt);
11337 : }
11338 : }
11339 :
11340 3019 : if (ls.slp_perm)
11341 : {
11342 0 : if (costing_p)
11343 : {
11344 0 : gcc_assert (ls.n_perms != -1U);
11345 0 : inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11346 : slp_node, 0, vect_body);
11347 : }
11348 : else
11349 : {
11350 0 : unsigned n_perms2;
11351 0 : vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf,
11352 : false, &n_perms2);
11353 0 : gcc_assert (ls.n_perms == n_perms2);
11354 : }
11355 : }
11356 :
11357 3019 : if (costing_p)
11358 : {
11359 2257 : if (dump_enabled_p ())
11360 315 : dump_printf_loc (MSG_NOTE, vect_location,
11361 : "vect_model_load_cost: inside_cost = %u, "
11362 : "prologue_cost = %u .\n",
11363 : inside_cost, prologue_cost);
11364 2257 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
11365 2257 : slp_node->data = new vect_load_store_data (std::move (ls));
11366 : }
11367 3019 : return true;
11368 3019 : }
11369 :
11370 599490 : aggr_type = vectype;
11371 599490 : if (!costing_p)
11372 : {
11373 166591 : dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
11374 : memory_access_type);
11375 166591 : dr_bump = vect_get_data_ptr_bump (vinfo, dr_info, aggr_type,
11376 : memory_access_type);
11377 : }
11378 :
11379 599490 : poly_uint64 group_elt = 0;
11380 599490 : unsigned int inside_cost = 0, prologue_cost = 0;
11381 : /* For costing some adjacent vector loads, we'd like to cost with
11382 : the total number of them once instead of cost each one by one. */
11383 599490 : unsigned int n_adjacent_loads = 0;
11384 :
11385 : /* 1. Create the vector or array pointer update chain. */
11386 599490 : if (!costing_p)
11387 : {
11388 166591 : bool simd_lane_access_p
11389 166591 : = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
11390 166591 : if (simd_lane_access_p
11391 1628 : && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
11392 1628 : && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
11393 1628 : && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
11394 1628 : && integer_zerop (DR_INIT (first_dr_info->dr))
11395 1628 : && alias_sets_conflict_p (get_alias_set (aggr_type),
11396 1628 : get_alias_set (TREE_TYPE (ref_type)))
11397 166591 : && (alignment_support_scheme == dr_aligned
11398 1628 : || alignment_support_scheme == dr_unaligned_supported))
11399 : {
11400 1628 : dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
11401 1628 : dataref_offset = build_int_cst (ref_type, 0);
11402 : }
11403 164963 : else if (diff_first_stmt_info)
11404 : {
11405 4384 : dataref_ptr
11406 4384 : = vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
11407 : aggr_type, at_loop, offset, &dummy,
11408 : gsi, NULL, simd_lane_access_p,
11409 : dr_increment);
11410 : /* Adjust the pointer by the difference to first_stmt. */
11411 4384 : data_reference_p ptrdr
11412 : = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
11413 4384 : tree diff = fold_convert (sizetype,
11414 : size_binop (MINUS_EXPR,
11415 : DR_INIT (first_dr_info->dr),
11416 : DR_INIT (ptrdr)));
11417 4384 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
11418 : stmt_info, diff);
11419 4384 : if (alignment_support_scheme == dr_explicit_realign)
11420 : {
11421 0 : msq = vect_setup_realignment (vinfo, first_stmt_info_for_drptr,
11422 : vectype, gsi,
11423 : &realignment_token,
11424 : alignment_support_scheme,
11425 : dataref_ptr, &at_loop);
11426 0 : gcc_assert (!compute_in_loop);
11427 : }
11428 : }
11429 : else
11430 160579 : dataref_ptr
11431 160579 : = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
11432 : at_loop,
11433 : offset, &dummy, gsi, NULL,
11434 : simd_lane_access_p, dr_increment);
11435 : }
11436 :
11437 599490 : auto_vec<tree> dr_chain;
11438 599490 : if (grouped_load || ls.slp_perm)
11439 61027 : dr_chain.create (vec_num);
11440 :
11441 599490 : gimple *new_stmt = NULL;
11442 1568048 : for (i = 0; i < vec_num; i++)
11443 : {
11444 968558 : tree final_mask = NULL_TREE;
11445 968558 : tree final_len = NULL_TREE;
11446 968558 : tree bias = NULL_TREE;
11447 :
11448 968558 : if (!costing_p)
11449 : {
11450 260564 : if (mask_node)
11451 651 : vec_mask = vec_masks[i];
11452 260564 : if (loop_masks)
11453 51 : final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
11454 : vec_num, vectype, i);
11455 260564 : if (vec_mask)
11456 651 : final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
11457 : final_mask, vec_mask, gsi);
11458 :
11459 260564 : if (i > 0)
11460 93973 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11461 : dr_bump);
11462 : }
11463 :
11464 : /* 2. Create the vector-load in the loop. */
11465 968558 : switch (alignment_support_scheme)
11466 : {
11467 968558 : case dr_aligned:
11468 968558 : case dr_unaligned_supported:
11469 968558 : {
11470 968558 : if (costing_p)
11471 : break;
11472 :
11473 260564 : unsigned int misalign;
11474 260564 : unsigned HOST_WIDE_INT align;
11475 260564 : align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
11476 260564 : if (alignment_support_scheme == dr_aligned)
11477 : misalign = 0;
11478 169466 : else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
11479 : {
11480 129598 : align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
11481 129598 : misalign = 0;
11482 : }
11483 : else
11484 39868 : misalign = misalignment;
11485 260564 : if (dataref_offset == NULL_TREE
11486 258438 : && TREE_CODE (dataref_ptr) == SSA_NAME)
11487 178212 : set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
11488 : misalign);
11489 260564 : align = least_bit_hwi (misalign | align);
11490 :
11491 : /* Compute IFN when LOOP_LENS or final_mask valid. */
11492 260564 : machine_mode vmode = TYPE_MODE (vectype);
11493 260564 : machine_mode new_vmode = vmode;
11494 260564 : internal_fn partial_ifn = IFN_LAST;
11495 260564 : if (loop_lens)
11496 : {
11497 0 : opt_machine_mode new_ovmode
11498 0 : = get_len_load_store_mode (vmode, true, &partial_ifn);
11499 0 : new_vmode = new_ovmode.require ();
11500 0 : unsigned factor
11501 0 : = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
11502 0 : final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
11503 : vec_num, vectype, i, factor, true);
11504 : }
11505 260564 : else if (final_mask)
11506 : {
11507 682 : if (!can_vec_mask_load_store_p (vmode,
11508 682 : TYPE_MODE
11509 : (TREE_TYPE (final_mask)),
11510 : true, &partial_ifn))
11511 0 : gcc_unreachable ();
11512 : }
11513 :
11514 260564 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11515 : {
11516 0 : if (!final_len)
11517 : {
11518 : /* Pass VF value to 'len' argument of
11519 : MASK_LEN_LOAD if LOOP_LENS is invalid. */
11520 0 : final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11521 : }
11522 0 : if (!final_mask)
11523 : {
11524 : /* Pass all ones value to 'mask' argument of
11525 : MASK_LEN_LOAD if final_mask is invalid. */
11526 0 : mask_vectype = truth_type_for (vectype);
11527 0 : final_mask = build_minus_one_cst (mask_vectype);
11528 : }
11529 : }
11530 260564 : if (final_len)
11531 : {
11532 0 : signed char biasval
11533 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
11534 0 : bias = build_int_cst (intQI_type_node, biasval);
11535 : }
11536 :
11537 0 : tree vec_els;
11538 :
11539 0 : if (final_len)
11540 : {
11541 0 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11542 0 : gcall *call;
11543 :
11544 : /* Need conversion if the vectype is punned by VnQI. */
11545 0 : els_vectype = vectype;
11546 0 : if (vmode != new_vmode)
11547 0 : els_vectype
11548 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11549 : new_vmode);
11550 0 : vec_els = vect_get_mask_load_else (maskload_elsval,
11551 : els_vectype);
11552 :
11553 0 : if (partial_ifn == IFN_MASK_LEN_LOAD)
11554 : {
11555 0 : if (type_mode_padding_p
11556 0 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11557 0 : need_zeroing = true;
11558 0 : call = gimple_build_call_internal (IFN_MASK_LEN_LOAD,
11559 : 6, dataref_ptr, ptr,
11560 : final_mask, vec_els,
11561 : final_len, bias);
11562 : }
11563 : else
11564 0 : call = gimple_build_call_internal (IFN_LEN_LOAD, 5,
11565 : dataref_ptr, ptr,
11566 : vec_els, final_len,
11567 : bias);
11568 0 : gimple_call_set_nothrow (call, true);
11569 0 : new_stmt = call;
11570 0 : data_ref = NULL_TREE;
11571 :
11572 : /* Need conversion if it's wrapped with VnQI. */
11573 0 : if (vmode != new_vmode)
11574 : {
11575 0 : tree new_vtype
11576 0 : = build_vector_type_for_mode (unsigned_intQI_type_node,
11577 : new_vmode);
11578 0 : tree var = vect_get_new_ssa_name (new_vtype,
11579 : vect_simple_var);
11580 0 : gimple_set_lhs (call, var);
11581 0 : vect_finish_stmt_generation (vinfo, stmt_info, call,
11582 : gsi);
11583 0 : tree op = build1 (VIEW_CONVERT_EXPR, vectype, var);
11584 0 : new_stmt = gimple_build_assign (vec_dest,
11585 : VIEW_CONVERT_EXPR, op);
11586 : }
11587 : }
11588 260564 : else if (final_mask)
11589 : {
11590 682 : tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
11591 682 : vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
11592 682 : if (type_mode_padding_p
11593 682 : && maskload_elsval != MASK_LOAD_ELSE_ZERO)
11594 0 : need_zeroing = true;
11595 682 : gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
11596 : dataref_ptr, ptr,
11597 : final_mask,
11598 : vec_els);
11599 682 : gimple_call_set_nothrow (call, true);
11600 682 : new_stmt = call;
11601 682 : data_ref = NULL_TREE;
11602 : }
11603 : else
11604 : {
11605 259882 : tree ltype = vectype;
11606 259882 : tree new_vtype = NULL_TREE;
11607 259882 : unsigned HOST_WIDE_INT gap = DR_GROUP_GAP (first_stmt_info);
11608 259882 : unsigned HOST_WIDE_INT dr_size
11609 259882 : = vect_get_scalar_dr_size (first_dr_info);
11610 259882 : poly_int64 off = 0;
11611 259882 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11612 1442 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
11613 259882 : unsigned int vect_align
11614 259882 : = vect_known_alignment_in_bytes (first_dr_info, vectype,
11615 259882 : off);
11616 : /* Try to use a single smaller load when we are about
11617 : to load excess elements compared to the unrolled
11618 : scalar loop. */
11619 259882 : if (known_gt ((i + 1) * nunits,
11620 : (group_size * vf - gap)))
11621 : {
11622 7378 : poly_uint64 remain = ((group_size * vf - gap) - i * nunits);
11623 7378 : if (known_ge ((i + 1) * nunits - (group_size * vf - gap),
11624 : nunits))
11625 : /* DR will be unused. */
11626 : ltype = NULL_TREE;
11627 2687 : else if (known_ge (vect_align,
11628 : tree_to_poly_uint64
11629 : (TYPE_SIZE_UNIT (vectype))))
11630 : /* Aligned access to excess elements is OK if
11631 : at least one element is accessed in the
11632 : scalar loop. */
11633 : ;
11634 2286 : else if (known_gt (vect_align,
11635 : ((nunits - remain) * dr_size)))
11636 : /* Aligned access to the gap area when there's
11637 : at least one element in it is OK. */
11638 : ;
11639 : else
11640 : {
11641 : /* remain should now be > 0 and < nunits. */
11642 2283 : unsigned num;
11643 2283 : if (known_ne (remain, 0u)
11644 2283 : && constant_multiple_p (nunits, remain, &num))
11645 : {
11646 1791 : tree ptype;
11647 1791 : new_vtype
11648 1791 : = vector_vector_composition_type (vectype, num,
11649 : &ptype);
11650 1791 : if (new_vtype)
11651 1791 : ltype = ptype;
11652 : }
11653 : /* Else use multiple loads or a masked load? */
11654 : /* For loop vectorization we now should have
11655 : an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11656 : set. */
11657 2283 : if (loop_vinfo)
11658 1686 : gcc_assert (new_vtype
11659 : || LOOP_VINFO_PEELING_FOR_GAPS
11660 : (loop_vinfo));
11661 : /* But still reduce the access size to the next
11662 : required power-of-two so peeling a single
11663 : scalar iteration is sufficient. */
11664 2283 : unsigned HOST_WIDE_INT cremain;
11665 2283 : if (remain.is_constant (&cremain))
11666 : {
11667 2283 : unsigned HOST_WIDE_INT cpart_size
11668 2283 : = 1 << ceil_log2 (cremain);
11669 2283 : if (known_gt (nunits, cpart_size)
11670 2283 : && constant_multiple_p (nunits, cpart_size,
11671 : &num))
11672 : {
11673 1803 : tree ptype;
11674 1803 : new_vtype
11675 3606 : = vector_vector_composition_type (vectype,
11676 1803 : num,
11677 : &ptype);
11678 1803 : if (new_vtype)
11679 1803 : ltype = ptype;
11680 : }
11681 : }
11682 : }
11683 : }
11684 259882 : tree offset = (dataref_offset ? dataref_offset
11685 257756 : : build_int_cst (ref_type, 0));
11686 259882 : if (!ltype)
11687 : ;
11688 255191 : else if (ltype != vectype
11689 255191 : && memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11690 : {
11691 25 : poly_uint64 gap_offset
11692 25 : = (tree_to_poly_uint64 (TYPE_SIZE_UNIT (vectype))
11693 25 : - tree_to_poly_uint64 (TYPE_SIZE_UNIT (ltype)));
11694 25 : tree gapcst = build_int_cstu (ref_type, gap_offset);
11695 25 : offset = size_binop (PLUS_EXPR, offset, gapcst);
11696 : }
11697 255191 : if (ltype)
11698 : {
11699 255191 : data_ref = fold_build2 (MEM_REF, ltype,
11700 : dataref_ptr, offset);
11701 255191 : if (alignment_support_scheme == dr_aligned
11702 255191 : && align >= TYPE_ALIGN_UNIT (ltype))
11703 : ;
11704 : else
11705 167806 : TREE_TYPE (data_ref)
11706 335612 : = build_aligned_type (TREE_TYPE (data_ref),
11707 : align * BITS_PER_UNIT);
11708 : }
11709 255191 : if (!ltype)
11710 4691 : data_ref = build_constructor (vectype, NULL);
11711 255191 : else if (ltype != vectype)
11712 : {
11713 1803 : vect_copy_ref_info (data_ref,
11714 1803 : DR_REF (first_dr_info->dr));
11715 1803 : tree tem = make_ssa_name (ltype);
11716 1803 : new_stmt = gimple_build_assign (tem, data_ref);
11717 1803 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11718 : gsi);
11719 1803 : data_ref = NULL;
11720 1803 : vec<constructor_elt, va_gc> *v;
11721 : /* We've computed 'num' above to statically two
11722 : or via constant_multiple_p. */
11723 1803 : unsigned num
11724 1803 : = (exact_div (tree_to_poly_uint64
11725 1803 : (TYPE_SIZE_UNIT (vectype)),
11726 : tree_to_poly_uint64
11727 1803 : (TYPE_SIZE_UNIT (ltype)))
11728 1803 : .to_constant ());
11729 1803 : vec_alloc (v, num);
11730 1803 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11731 : {
11732 62 : while (--num)
11733 62 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11734 : build_zero_cst (ltype));
11735 25 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11736 : }
11737 : else
11738 : {
11739 1778 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, tem);
11740 1778 : while (--num)
11741 3928 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
11742 : build_zero_cst (ltype));
11743 : }
11744 1803 : gcc_assert (new_vtype != NULL_TREE);
11745 1803 : if (new_vtype == vectype)
11746 1771 : new_stmt
11747 1771 : = gimple_build_assign (vec_dest,
11748 : build_constructor (vectype, v));
11749 : else
11750 : {
11751 32 : tree new_vname = make_ssa_name (new_vtype);
11752 32 : new_stmt
11753 32 : = gimple_build_assign (new_vname,
11754 : build_constructor (new_vtype,
11755 : v));
11756 32 : vect_finish_stmt_generation (vinfo, stmt_info,
11757 : new_stmt, gsi);
11758 32 : new_stmt
11759 32 : = gimple_build_assign (vec_dest,
11760 : build1 (VIEW_CONVERT_EXPR,
11761 : vectype, new_vname));
11762 : }
11763 : }
11764 : }
11765 : break;
11766 : }
11767 0 : case dr_explicit_realign:
11768 0 : {
11769 0 : if (costing_p)
11770 : break;
11771 0 : tree ptr, bump;
11772 :
11773 0 : tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
11774 :
11775 0 : if (compute_in_loop)
11776 0 : msq = vect_setup_realignment (vinfo, first_stmt_info, vectype,
11777 : gsi, &realignment_token,
11778 : dr_explicit_realign,
11779 : dataref_ptr, NULL);
11780 :
11781 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11782 0 : ptr = copy_ssa_name (dataref_ptr);
11783 : else
11784 0 : ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
11785 : // For explicit realign the target alignment should be
11786 : // known at compile time.
11787 0 : unsigned HOST_WIDE_INT align
11788 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11789 0 : new_stmt = gimple_build_assign (ptr, BIT_AND_EXPR, dataref_ptr,
11790 : build_int_cst
11791 0 : (TREE_TYPE (dataref_ptr),
11792 0 : -(HOST_WIDE_INT) align));
11793 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11794 0 : data_ref = build2 (MEM_REF, vectype,
11795 : ptr, build_int_cst (ref_type, 0));
11796 0 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11797 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11798 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11799 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11800 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11801 0 : gimple_move_vops (new_stmt, stmt_info->stmt);
11802 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11803 0 : msq = new_temp;
11804 :
11805 0 : bump = size_binop (MULT_EXPR, vs, TYPE_SIZE_UNIT (elem_type));
11806 0 : bump = size_binop (MINUS_EXPR, bump, size_one_node);
11807 0 : ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11808 0 : new_stmt = gimple_build_assign (NULL_TREE, BIT_AND_EXPR, ptr,
11809 0 : build_int_cst (TREE_TYPE (ptr),
11810 0 : -(HOST_WIDE_INT) align));
11811 0 : if (TREE_CODE (ptr) == SSA_NAME)
11812 0 : ptr = copy_ssa_name (ptr, new_stmt);
11813 : else
11814 0 : ptr = make_ssa_name (TREE_TYPE (ptr), new_stmt);
11815 0 : gimple_assign_set_lhs (new_stmt, ptr);
11816 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11817 0 : data_ref = build2 (MEM_REF, vectype,
11818 : ptr, build_int_cst (ref_type, 0));
11819 0 : break;
11820 : }
11821 0 : case dr_explicit_realign_optimized:
11822 0 : {
11823 0 : if (costing_p)
11824 : break;
11825 0 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
11826 0 : new_temp = copy_ssa_name (dataref_ptr);
11827 : else
11828 0 : new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
11829 : // We should only be doing this if we know the target
11830 : // alignment at compile time.
11831 0 : unsigned HOST_WIDE_INT align
11832 0 : = DR_TARGET_ALIGNMENT (first_dr_info).to_constant ();
11833 0 : new_stmt = gimple_build_assign (new_temp, BIT_AND_EXPR, dataref_ptr,
11834 0 : build_int_cst (TREE_TYPE (dataref_ptr),
11835 0 : -(HOST_WIDE_INT) align));
11836 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11837 0 : data_ref = build2 (MEM_REF, vectype, new_temp,
11838 : build_int_cst (ref_type, 0));
11839 0 : break;
11840 : }
11841 0 : default:
11842 0 : gcc_unreachable ();
11843 : }
11844 :
11845 : /* One common place to cost the above vect load for different
11846 : alignment support schemes. */
11847 968558 : if (costing_p)
11848 : {
11849 : /* For the prologue cost for realign,
11850 : we only need to count it once for the whole group. */
11851 707994 : bool first_stmt_info_p = first_stmt_info == stmt_info;
11852 707994 : bool add_realign_cost = first_stmt_info_p && i == 0;
11853 707994 : if (memory_access_type == VMAT_CONTIGUOUS
11854 707994 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11855 : {
11856 : /* Leave realign cases alone to keep them simple. */
11857 707994 : if (alignment_support_scheme == dr_explicit_realign_optimized
11858 : || alignment_support_scheme == dr_explicit_realign)
11859 0 : vect_get_load_cost (vinfo, stmt_info, slp_node, 1,
11860 : alignment_support_scheme, misalignment,
11861 : add_realign_cost, &inside_cost,
11862 : &prologue_cost, cost_vec, cost_vec,
11863 : true);
11864 : else
11865 707994 : n_adjacent_loads++;
11866 : }
11867 : }
11868 : else
11869 : {
11870 260564 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11871 : /* DATA_REF is null if we've already built the statement. */
11872 260564 : if (data_ref)
11873 : {
11874 258079 : vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
11875 258079 : new_stmt = gimple_build_assign (vec_dest, data_ref);
11876 : }
11877 :
11878 521128 : new_temp = (need_zeroing
11879 260564 : ? make_ssa_name (vectype)
11880 260564 : : make_ssa_name (vec_dest, new_stmt));
11881 260564 : gimple_set_lhs (new_stmt, new_temp);
11882 260564 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11883 :
11884 : /* If we need to explicitly zero inactive elements emit a
11885 : VEC_COND_EXPR that does so. */
11886 260564 : if (need_zeroing)
11887 : {
11888 0 : vec_els = vect_get_mask_load_else (MASK_LOAD_ELSE_ZERO,
11889 : vectype);
11890 :
11891 0 : tree new_temp2 = make_ssa_name (vec_dest, new_stmt);
11892 0 : new_stmt = gimple_build_assign (new_temp2, VEC_COND_EXPR,
11893 : final_mask, new_temp, vec_els);
11894 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
11895 : gsi);
11896 0 : new_temp = new_temp2;
11897 : }
11898 : }
11899 :
11900 : /* 3. Handle explicit realignment if necessary/supported.
11901 : Create in loop:
11902 : vec_dest = realign_load (msq, lsq, realignment_token) */
11903 968558 : if (!costing_p
11904 260564 : && (alignment_support_scheme == dr_explicit_realign_optimized
11905 : || alignment_support_scheme == dr_explicit_realign))
11906 : {
11907 0 : lsq = gimple_assign_lhs (new_stmt);
11908 0 : if (!realignment_token)
11909 0 : realignment_token = dataref_ptr;
11910 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
11911 0 : new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR, msq,
11912 : lsq, realignment_token);
11913 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
11914 0 : gimple_assign_set_lhs (new_stmt, new_temp);
11915 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
11916 :
11917 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
11918 : {
11919 0 : gcc_assert (phi);
11920 0 : if (i == vec_num - 1)
11921 0 : add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
11922 : UNKNOWN_LOCATION);
11923 : msq = lsq;
11924 : }
11925 : }
11926 :
11927 968558 : if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
11928 : {
11929 5943 : if (costing_p)
11930 4501 : inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
11931 : slp_node, 0, vect_body);
11932 : else
11933 : {
11934 1442 : tree perm_mask = perm_mask_for_reverse (vectype);
11935 1442 : new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
11936 : perm_mask, stmt_info, gsi);
11937 1442 : new_stmt = SSA_NAME_DEF_STMT (new_temp);
11938 : }
11939 : }
11940 :
11941 : /* Collect vector loads and later create their permutation in
11942 : vect_transform_slp_perm_load. */
11943 968558 : if (!costing_p && (grouped_load || ls.slp_perm))
11944 76495 : dr_chain.quick_push (new_temp);
11945 :
11946 : /* Store vector loads in the corresponding SLP_NODE. */
11947 260564 : if (!costing_p && !ls.slp_perm)
11948 184069 : slp_node->push_vec_def (new_stmt);
11949 :
11950 : /* With SLP permutation we load the gaps as well, without
11951 : we need to skip the gaps after we manage to fully load
11952 : all elements. group_gap_adj is DR_GROUP_SIZE here. */
11953 968558 : group_elt += nunits;
11954 968558 : if (!costing_p
11955 260564 : && maybe_ne (group_gap_adj, 0U)
11956 46433 : && !ls.slp_perm
11957 988526 : && known_eq (group_elt, group_size - group_gap_adj))
11958 : {
11959 16625 : poly_wide_int bump_val
11960 16625 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11961 16625 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11962 0 : bump_val = -bump_val;
11963 16625 : tree bump = wide_int_to_tree (sizetype, bump_val);
11964 16625 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info,
11965 : bump);
11966 16625 : group_elt = 0;
11967 16625 : }
11968 : }
11969 : /* Bump the vector pointer to account for a gap or for excess
11970 : elements loaded for a permuted SLP load. */
11971 599490 : if (!costing_p
11972 166591 : && maybe_ne (group_gap_adj, 0U)
11973 616929 : && ls.slp_perm)
11974 : {
11975 814 : poly_wide_int bump_val
11976 814 : = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj);
11977 814 : if (tree_int_cst_sgn (vect_dr_behavior (vinfo, dr_info)->step) == -1)
11978 9 : bump_val = -bump_val;
11979 814 : tree bump = wide_int_to_tree (sizetype, bump_val);
11980 814 : dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi, stmt_info, bump);
11981 814 : }
11982 :
11983 599490 : if (ls.slp_perm)
11984 : {
11985 : /* For SLP we know we've seen all possible uses of dr_chain so
11986 : direct vect_transform_slp_perm_load to DCE the unused parts.
11987 : ??? This is a hack to prevent compile-time issues as seen
11988 : in PR101120 and friends. */
11989 61027 : if (costing_p)
11990 : {
11991 43096 : gcc_assert (ls.n_perms != -1U && ls.n_loads != -1U);
11992 43096 : if (ls.n_perms != 0)
11993 42561 : inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm,
11994 : slp_node, 0, vect_body);
11995 43096 : if (n_adjacent_loads > 0)
11996 43096 : n_adjacent_loads = ls.n_loads;
11997 : }
11998 : else
11999 : {
12000 17931 : unsigned n_perms2, n_loads2;
12001 17931 : bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
12002 : gsi, vf, false, &n_perms2,
12003 : &n_loads2, true);
12004 17931 : gcc_assert (ok && ls.n_perms == n_perms2 && ls.n_loads == n_loads2);
12005 : }
12006 : }
12007 :
12008 599490 : if (costing_p)
12009 : {
12010 432899 : gcc_assert (memory_access_type == VMAT_CONTIGUOUS
12011 : || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
12012 432899 : if (n_adjacent_loads > 0)
12013 432899 : vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
12014 : alignment_support_scheme, misalignment, false,
12015 : &inside_cost, &prologue_cost, cost_vec, cost_vec,
12016 : true);
12017 432899 : if (dump_enabled_p ())
12018 24676 : dump_printf_loc (MSG_NOTE, vect_location,
12019 : "vect_model_load_cost: inside_cost = %u, "
12020 : "prologue_cost = %u .\n",
12021 : inside_cost, prologue_cost);
12022 432899 : SLP_TREE_TYPE (slp_node) = load_vec_info_type;
12023 432899 : slp_node->data = new vect_load_store_data (std::move (ls));
12024 : }
12025 :
12026 599490 : return true;
12027 1344445 : }
12028 :
12029 : /* vectorizable_condition.
12030 :
12031 : Check if STMT_INFO is conditional modify expression that can be vectorized.
12032 : If COST_VEC is passed, calculate costs but don't change anything,
12033 : otherwise, vectorize STMT_INFO: create a vectorized stmt using
12034 : VEC_COND_EXPR to replace it, and insert it at GSI.
12035 :
12036 : When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12037 :
12038 : Return true if STMT_INFO is vectorizable in this way. */
12039 :
12040 : static bool
12041 727360 : vectorizable_condition (vec_info *vinfo,
12042 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12043 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12044 : {
12045 727360 : tree scalar_dest = NULL_TREE;
12046 727360 : tree vec_dest = NULL_TREE;
12047 727360 : tree then_clause, else_clause;
12048 727360 : tree vec_cond_lhs = NULL_TREE;
12049 727360 : tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
12050 727360 : tree vec_compare;
12051 727360 : tree new_temp;
12052 727360 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
12053 727360 : enum vect_def_type dts[4]
12054 : = {vect_unknown_def_type, vect_unknown_def_type,
12055 : vect_unknown_def_type, vect_unknown_def_type};
12056 727360 : enum tree_code code;
12057 727360 : int i;
12058 727360 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12059 727360 : vec<tree> vec_oprnds0 = vNULL;
12060 727360 : vec<tree> vec_oprnds2 = vNULL;
12061 727360 : vec<tree> vec_oprnds3 = vNULL;
12062 727360 : tree vec_cmp_type;
12063 :
12064 727360 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12065 : return false;
12066 :
12067 : /* Is vectorizable conditional operation? */
12068 727360 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12069 416096 : if (!stmt)
12070 : return false;
12071 :
12072 416096 : code = gimple_assign_rhs_code (stmt);
12073 416096 : if (code != COND_EXPR)
12074 : return false;
12075 :
12076 35739 : int reduc_index = SLP_TREE_REDUC_IDX (slp_node);
12077 35739 : vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
12078 35739 : bool nested_cycle_p = false;
12079 35739 : bool for_reduction = vect_is_reduction (stmt_info);
12080 35739 : if (for_reduction)
12081 : {
12082 623 : if (SLP_TREE_LANES (slp_node) > 1)
12083 : return false;
12084 : /* ??? With a reduction path we do not get at the reduction info from
12085 : every stmt, use the conservative default setting then. */
12086 703 : if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
12087 : {
12088 605 : vect_reduc_info reduc_info
12089 605 : = info_for_reduction (loop_vinfo, slp_node);
12090 605 : reduction_type = VECT_REDUC_INFO_TYPE (reduc_info);
12091 605 : nested_cycle_p = nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12092 : stmt_info);
12093 : }
12094 : }
12095 : else
12096 : {
12097 35116 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12098 : return false;
12099 : }
12100 :
12101 35739 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12102 35739 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12103 :
12104 35739 : int vec_num = vect_get_num_copies (vinfo, slp_node);
12105 :
12106 35739 : slp_tree slp_cond;
12107 35739 : tree cond_expr = gimple_assign_rhs1 (stmt);
12108 35739 : gcc_assert (! COMPARISON_CLASS_P (cond_expr));
12109 35739 : if (TREE_CODE (cond_expr) != SSA_NAME
12110 35739 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond_expr))
12111 35739 : || !vect_is_simple_use (vinfo, slp_node, 0,
12112 : &slp_cond, &dts[0], &vec_cmp_type)
12113 35739 : || !vec_cmp_type
12114 71461 : || !VECTOR_BOOLEAN_TYPE_P (vec_cmp_type))
12115 : return false;
12116 :
12117 35722 : slp_tree then_slp_node, else_slp_node;
12118 35722 : if (!vect_is_simple_use (vinfo, slp_node, 1,
12119 : &then_clause, &then_slp_node, &dts[2], &vectype1))
12120 : return false;
12121 35722 : if (!vect_is_simple_use (vinfo, slp_node, 2,
12122 : &else_clause, &else_slp_node, &dts[3], &vectype2))
12123 : return false;
12124 :
12125 35722 : if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
12126 : return false;
12127 :
12128 35722 : if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
12129 : return false;
12130 :
12131 35710 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
12132 71420 : TYPE_VECTOR_SUBPARTS (vec_cmp_type)))
12133 : return false;
12134 :
12135 : /* For conditional reductions, the "then" value needs to be the candidate
12136 : value calculated by this iteration while the "else" value needs to be
12137 : the result carried over from previous iterations. If the COND_EXPR
12138 : is the other way around, we need to swap it. */
12139 35710 : bool must_invert_cmp_result = false;
12140 35710 : if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
12141 : {
12142 727360 : must_invert_cmp_result = true;
12143 : /* ??? The vectorized operand query below doesn't allow swapping
12144 : this way for SLP. */
12145 : return false;
12146 : /* std::swap (then_clause, else_clause); */
12147 : }
12148 :
12149 35710 : if (cost_vec)
12150 : {
12151 27036 : vect_cost_for_stmt kind = vector_stmt;
12152 27036 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12153 : /* Count one reduction-like operation per vector. */
12154 : kind = vec_to_scalar;
12155 27036 : else if (!expand_vec_cond_expr_p (vectype, vec_cmp_type))
12156 : return false;
12157 :
12158 27022 : if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
12159 : vec_cmp_type)
12160 27022 : || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
12161 54044 : || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
12162 : {
12163 0 : if (dump_enabled_p ())
12164 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12165 : "incompatible vector types for invariants\n");
12166 : return false;
12167 : }
12168 :
12169 27022 : if (loop_vinfo && for_reduction
12170 456 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12171 : {
12172 77 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12173 : {
12174 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12175 : vectype, OPTIMIZE_FOR_SPEED))
12176 0 : vect_record_loop_len (loop_vinfo,
12177 : &LOOP_VINFO_LENS (loop_vinfo),
12178 : vec_num, vectype, 1);
12179 : else
12180 0 : vect_record_loop_mask (loop_vinfo,
12181 : &LOOP_VINFO_MASKS (loop_vinfo),
12182 : vec_num, vectype, NULL);
12183 : }
12184 : /* Extra inactive lanes should be safe for vect_nested_cycle. */
12185 77 : else if (!nested_cycle_p)
12186 : {
12187 77 : if (dump_enabled_p ())
12188 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12189 : "conditional reduction prevents the use"
12190 : " of partial vectors.\n");
12191 77 : LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
12192 : }
12193 : }
12194 :
12195 27022 : SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
12196 27022 : vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
12197 27022 : return true;
12198 : }
12199 :
12200 : /* Transform. */
12201 :
12202 : /* Handle def. */
12203 8674 : scalar_dest = gimple_assign_lhs (stmt);
12204 8674 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12205 8674 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
12206 :
12207 8674 : bool swap_cond_operands = false;
12208 :
12209 : /* See whether another part of the vectorized code applies a loop
12210 : mask to the condition, or to its inverse. */
12211 :
12212 8674 : vec_loop_masks *masks = NULL;
12213 8674 : vec_loop_lens *lens = NULL;
12214 8674 : if (loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12215 : {
12216 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12217 0 : lens = &LOOP_VINFO_LENS (loop_vinfo);
12218 : }
12219 8674 : else if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
12220 : {
12221 3 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12222 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12223 : else
12224 : {
12225 3 : scalar_cond_masked_key cond (cond_expr, 1);
12226 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12227 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12228 : else
12229 : {
12230 3 : bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
12231 3 : tree_code orig_code = cond.code;
12232 3 : cond.code = invert_tree_comparison (cond.code, honor_nans);
12233 : /* Try the inverse of the current mask. We check if the
12234 : inverse mask is live and if so we generate a negate of
12235 : the current mask such that we still honor NaNs. */
12236 3 : cond.inverted_p = true;
12237 3 : cond.code = orig_code;
12238 3 : if (loop_vinfo->scalar_cond_masked_set.contains (cond))
12239 : {
12240 0 : masks = &LOOP_VINFO_MASKS (loop_vinfo);
12241 0 : swap_cond_operands = true;
12242 0 : must_invert_cmp_result = true;
12243 : }
12244 : }
12245 : }
12246 : }
12247 :
12248 : /* Handle cond expr. */
12249 8674 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds2,
12250 : reduction_type != EXTRACT_LAST_REDUCTION, &vec_oprnds3);
12251 :
12252 8674 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12253 0 : vec_else_clause = else_clause;
12254 :
12255 : /* Arguments are ready. Create the new vector stmt. */
12256 20392 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
12257 : {
12258 11718 : vec_then_clause = vec_oprnds2[i];
12259 11718 : if (reduction_type != EXTRACT_LAST_REDUCTION)
12260 11718 : vec_else_clause = vec_oprnds3[i];
12261 :
12262 11718 : if (swap_cond_operands)
12263 0 : std::swap (vec_then_clause, vec_else_clause);
12264 :
12265 11718 : vec_compare = vec_cond_lhs;
12266 :
12267 : /* If we decided to apply a loop mask to the result of the vector
12268 : comparison, AND the comparison with the mask now. Later passes
12269 : should then be able to reuse the AND results between multiple
12270 : vector statements.
12271 :
12272 : For example:
12273 : for (int i = 0; i < 100; ++i)
12274 : x[i] = y[i] ? z[i] : 10;
12275 :
12276 : results in following optimized GIMPLE:
12277 :
12278 : mask__35.8_43 = vect__4.7_41 != { 0, ... };
12279 : vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12280 : _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12281 : vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12282 : vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12283 : vect_iftmp.11_47, { 10, ... }>;
12284 :
12285 : instead of using a masked and unmasked forms of
12286 : vec != { 0, ... } (masked in the MASK_LOAD,
12287 : unmasked in the VEC_COND_EXPR). */
12288 :
12289 : /* Force vec_compare to be an SSA_NAME rather than a comparison,
12290 : in cases where that's necessary. */
12291 :
12292 11718 : tree len = NULL_TREE, bias = NULL_TREE;
12293 11718 : if (masks || lens || reduction_type == EXTRACT_LAST_REDUCTION)
12294 : {
12295 0 : if (!is_gimple_val (vec_compare))
12296 : {
12297 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12298 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12299 : vec_compare);
12300 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12301 0 : vec_compare = vec_compare_name;
12302 : }
12303 :
12304 0 : if (must_invert_cmp_result)
12305 : {
12306 0 : tree vec_compare_name = make_ssa_name (vec_cmp_type);
12307 0 : gassign *new_stmt = gimple_build_assign (vec_compare_name,
12308 : BIT_NOT_EXPR,
12309 : vec_compare);
12310 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12311 0 : vec_compare = vec_compare_name;
12312 : }
12313 :
12314 0 : if (direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST,
12315 : vectype, OPTIMIZE_FOR_SPEED))
12316 : {
12317 0 : if (lens)
12318 : {
12319 : /* ??? Do we really want the adjusted LEN here? Isn't this
12320 : based on number of elements? */
12321 0 : len = vect_get_loop_len (loop_vinfo, gsi, lens,
12322 : vec_num, vectype, i, 1, true);
12323 0 : signed char biasval
12324 0 : = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
12325 0 : bias = build_int_cst (intQI_type_node, biasval);
12326 : }
12327 : else
12328 : {
12329 0 : len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
12330 0 : bias = build_int_cst (intQI_type_node, 0);
12331 : }
12332 : }
12333 0 : if (masks)
12334 : {
12335 0 : tree loop_mask
12336 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12337 : vectype, i);
12338 0 : tree tmp2 = make_ssa_name (vec_cmp_type);
12339 0 : gassign *g
12340 0 : = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
12341 : loop_mask);
12342 0 : vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
12343 0 : vec_compare = tmp2;
12344 : }
12345 : }
12346 :
12347 0 : gimple *new_stmt;
12348 0 : if (reduction_type == EXTRACT_LAST_REDUCTION)
12349 : {
12350 0 : gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt;
12351 0 : tree lhs = gimple_get_lhs (old_stmt);
12352 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12353 0 : lhs = copy_ssa_name (lhs);
12354 0 : if (len)
12355 0 : new_stmt = gimple_build_call_internal
12356 0 : (IFN_LEN_FOLD_EXTRACT_LAST, 5, vec_else_clause, vec_compare,
12357 : vec_then_clause, len, bias);
12358 : else
12359 0 : new_stmt = gimple_build_call_internal
12360 0 : (IFN_FOLD_EXTRACT_LAST, 3, vec_else_clause, vec_compare,
12361 : vec_then_clause);
12362 0 : gimple_call_set_lhs (new_stmt, lhs);
12363 0 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12364 0 : if ((unsigned)i != vec_oprnds0.length () - 1)
12365 : {
12366 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12367 0 : vec_else_clause = lhs;
12368 : }
12369 0 : else if (old_stmt == gsi_stmt (*gsi))
12370 0 : vect_finish_replace_stmt (vinfo, stmt_info, new_stmt);
12371 : else
12372 : {
12373 : /* In this case we're moving the definition to later in the
12374 : block. That doesn't matter because the only uses of the
12375 : lhs are in phi statements. */
12376 0 : gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt);
12377 0 : gsi_remove (&old_gsi, true);
12378 0 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12379 : }
12380 : }
12381 : else
12382 : {
12383 11718 : new_temp = make_ssa_name (vec_dest);
12384 11718 : new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare,
12385 : vec_then_clause, vec_else_clause);
12386 11718 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12387 : }
12388 11718 : slp_node->push_vec_def (new_stmt);
12389 : }
12390 :
12391 8674 : vec_oprnds0.release ();
12392 8674 : vec_oprnds2.release ();
12393 8674 : vec_oprnds3.release ();
12394 :
12395 8674 : return true;
12396 : }
12397 :
12398 : /* Helper of vectorizable_comparison.
12399 :
12400 : Check if STMT_INFO is comparison expression CODE that can be vectorized.
12401 : If COST_VEC is passed, calculate costs but don't change anything,
12402 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12403 : it at GSI.
12404 :
12405 : Return true if STMT_INFO is vectorizable in this way. */
12406 :
12407 : static bool
12408 388888 : vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
12409 : stmt_vec_info stmt_info, tree_code code,
12410 : gimple_stmt_iterator *gsi,
12411 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12412 : {
12413 388888 : tree lhs, rhs1, rhs2;
12414 388888 : tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
12415 388888 : tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
12416 388888 : tree new_temp;
12417 388888 : enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
12418 388888 : poly_uint64 nunits;
12419 388888 : enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
12420 388888 : int i;
12421 388888 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12422 388888 : vec<tree> vec_oprnds0 = vNULL;
12423 388888 : vec<tree> vec_oprnds1 = vNULL;
12424 388888 : tree mask_type;
12425 388888 : tree mask = NULL_TREE;
12426 :
12427 388888 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12428 : return false;
12429 :
12430 388888 : if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
12431 : return false;
12432 :
12433 173366 : mask_type = vectype;
12434 173366 : nunits = TYPE_VECTOR_SUBPARTS (vectype);
12435 :
12436 173366 : if (TREE_CODE_CLASS (code) != tcc_comparison)
12437 : return false;
12438 :
12439 171414 : slp_tree slp_rhs1, slp_rhs2;
12440 171414 : if (!vect_is_simple_use (vinfo, slp_node,
12441 : 0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
12442 : return false;
12443 :
12444 171414 : if (!vect_is_simple_use (vinfo, slp_node,
12445 : 1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
12446 : return false;
12447 :
12448 133728 : if (vectype1 && vectype2
12449 248328 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
12450 76914 : TYPE_VECTOR_SUBPARTS (vectype2)))
12451 16 : return false;
12452 :
12453 171398 : vectype = vectype1 ? vectype1 : vectype2;
12454 :
12455 : /* Invariant comparison. */
12456 171398 : if (!vectype)
12457 : {
12458 33150 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), slp_node);
12459 33150 : if (!vectype || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
12460 : return false;
12461 : }
12462 138248 : else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
12463 : return false;
12464 :
12465 : /* Can't compare mask and non-mask types. */
12466 133712 : if (vectype1 && vectype2
12467 401456 : && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
12468 : return false;
12469 :
12470 : /* We cannot compare non-mode precision _BitInt types. Unlike bool
12471 : or bit-precision INTEGER_TYPE the padding bit values are target
12472 : dependent and possibly undefined. */
12473 171386 : if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE
12474 171386 : && !type_has_mode_precision_p (TREE_TYPE (rhs1)))
12475 : return false;
12476 :
12477 : /* Boolean values may have another representation in vectors
12478 : and therefore we prefer bit operations over comparison for
12479 : them (which also works for scalar masks). We store opcodes
12480 : to use in bitop1 and bitop2. Statement is vectorized as
12481 : BITOP2 (rhs1 BITOP1 rhs2) or
12482 : rhs1 BITOP2 (BITOP1 rhs2)
12483 : depending on bitop1 and bitop2 arity. */
12484 171300 : bool swap_p = false;
12485 171300 : if (VECTOR_BOOLEAN_TYPE_P (vectype))
12486 : {
12487 765 : if (code == GT_EXPR)
12488 : {
12489 : bitop1 = BIT_NOT_EXPR;
12490 : bitop2 = BIT_AND_EXPR;
12491 : }
12492 : else if (code == GE_EXPR)
12493 : {
12494 : bitop1 = BIT_NOT_EXPR;
12495 : bitop2 = BIT_IOR_EXPR;
12496 : }
12497 : else if (code == LT_EXPR)
12498 : {
12499 : bitop1 = BIT_NOT_EXPR;
12500 : bitop2 = BIT_AND_EXPR;
12501 : swap_p = true;
12502 : }
12503 : else if (code == LE_EXPR)
12504 : {
12505 : bitop1 = BIT_NOT_EXPR;
12506 : bitop2 = BIT_IOR_EXPR;
12507 : swap_p = true;
12508 : }
12509 : else
12510 : {
12511 : bitop1 = BIT_XOR_EXPR;
12512 : if (code == EQ_EXPR)
12513 : bitop2 = BIT_NOT_EXPR;
12514 : }
12515 : }
12516 :
12517 171300 : if (cost_vec)
12518 : {
12519 158613 : if (bitop1 == NOP_EXPR)
12520 : {
12521 157997 : if (!expand_vec_cmp_expr_p (vectype, mask_type, code))
12522 : return false;
12523 : }
12524 : else
12525 : {
12526 616 : machine_mode mode = TYPE_MODE (vectype);
12527 616 : optab optab;
12528 :
12529 616 : optab = optab_for_tree_code (bitop1, vectype, optab_default);
12530 616 : if (!optab || !can_implement_p (optab, mode))
12531 : return false;
12532 :
12533 616 : if (bitop2 != NOP_EXPR)
12534 : {
12535 97 : optab = optab_for_tree_code (bitop2, vectype, optab_default);
12536 97 : if (!optab || !can_implement_p (optab, mode))
12537 : return false;
12538 : }
12539 : }
12540 :
12541 : /* Put types on constant and invariant SLP children. */
12542 148386 : if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
12543 148386 : || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
12544 : {
12545 2 : if (dump_enabled_p ())
12546 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12547 : "incompatible vector types for invariants\n");
12548 : return false;
12549 : }
12550 :
12551 148384 : vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
12552 : slp_node, cost_vec);
12553 148384 : return true;
12554 : }
12555 :
12556 : /* Transform. */
12557 :
12558 : /* Handle def. */
12559 12687 : lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
12560 12687 : if (lhs)
12561 12687 : mask = vect_create_destination_var (lhs, mask_type);
12562 :
12563 12687 : vect_get_vec_defs (vinfo, slp_node, true, &vec_oprnds0, true, &vec_oprnds1);
12564 12687 : if (swap_p)
12565 60 : std::swap (vec_oprnds0, vec_oprnds1);
12566 :
12567 : /* Arguments are ready. Create the new vector stmt. */
12568 31878 : FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
12569 : {
12570 19191 : gimple *new_stmt;
12571 19191 : vec_rhs2 = vec_oprnds1[i];
12572 :
12573 19191 : if (lhs)
12574 19191 : new_temp = make_ssa_name (mask);
12575 : else
12576 0 : new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
12577 19191 : if (bitop1 == NOP_EXPR)
12578 : {
12579 19032 : new_stmt = gimple_build_assign (new_temp, code,
12580 : vec_rhs1, vec_rhs2);
12581 19032 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12582 : }
12583 : else
12584 : {
12585 159 : if (bitop1 == BIT_NOT_EXPR)
12586 86 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
12587 : else
12588 73 : new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
12589 : vec_rhs2);
12590 159 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12591 159 : if (bitop2 != NOP_EXPR)
12592 : {
12593 86 : tree res = make_ssa_name (mask);
12594 86 : if (bitop2 == BIT_NOT_EXPR)
12595 0 : new_stmt = gimple_build_assign (res, bitop2, new_temp);
12596 : else
12597 86 : new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
12598 : new_temp);
12599 86 : vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
12600 : }
12601 : }
12602 19191 : slp_node->push_vec_def (new_stmt);
12603 : }
12604 :
12605 12687 : vec_oprnds0.release ();
12606 12687 : vec_oprnds1.release ();
12607 :
12608 12687 : return true;
12609 : }
12610 :
12611 : /* vectorizable_comparison.
12612 :
12613 : Check if STMT_INFO is comparison expression that can be vectorized.
12614 : If COST_VEC is passed, calculate costs but don't change anything,
12615 : otherwise, vectorize STMT_INFO: create a vectorized comparison, and insert
12616 : it at GSI.
12617 :
12618 : Return true if STMT_INFO is vectorizable in this way. */
12619 :
12620 : static bool
12621 704351 : vectorizable_comparison (vec_info *vinfo,
12622 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
12623 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12624 : {
12625 704351 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
12626 :
12627 704351 : if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
12628 : return false;
12629 :
12630 704351 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
12631 : return false;
12632 :
12633 459460 : gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
12634 386218 : if (!stmt)
12635 : return false;
12636 :
12637 386218 : enum tree_code code = gimple_assign_rhs_code (stmt);
12638 386218 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12639 386218 : if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
12640 : slp_node, cost_vec))
12641 : return false;
12642 :
12643 158401 : if (cost_vec)
12644 145714 : SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
12645 :
12646 : return true;
12647 : }
12648 :
12649 : /* Check to see if the target supports any of the compare and branch optabs for
12650 : vectors with MODE as these would be required when expanding. */
12651 : static bool
12652 66034 : supports_vector_compare_and_branch (loop_vec_info loop_vinfo, machine_mode mode)
12653 : {
12654 66034 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12655 66034 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12656 :
12657 : /* The vectorizer only produces vec_cbranch_any_optab directly. So only
12658 : check for support for that or vec_cbranch_any_optab when masked.
12659 : We can't produce vcond_cbranch_any directly from the vectorizer as we
12660 : want to keep gimple_cond as the GIMPLE representation. But we'll fold
12661 : it in expand. For that reason we require a backend to support the
12662 : unconditional vector cbranch optab if they support the conditional one,
12663 : which is just an optimization on the unconditional one. */
12664 66034 : if (masked_loop_p
12665 66034 : && direct_optab_handler (cond_vec_cbranch_any_optab, mode)
12666 : != CODE_FOR_nothing)
12667 : return true;
12668 66034 : else if (len_loop_p
12669 66034 : && direct_optab_handler (cond_len_vec_cbranch_any_optab, mode)
12670 : != CODE_FOR_nothing)
12671 : return true;
12672 66034 : else if (!masked_loop_p && !len_loop_p
12673 132068 : && direct_optab_handler (vec_cbranch_any_optab, mode)
12674 : != CODE_FOR_nothing)
12675 : return true;
12676 :
12677 : /* The target can implement cbranch to distinguish between boolean vector
12678 : types and data types if they don't have a different mode for both. */
12679 66034 : return direct_optab_handler (cbranch_optab, mode) != CODE_FOR_nothing;
12680 : }
12681 :
12682 : /* Determine the type to use for early break vectorization's scalar IV. If
12683 : no type is possible return false. */
12684 :
12685 : static bool
12686 2670 : vect_compute_type_for_early_break_scalar_iv (loop_vec_info loop_vinfo)
12687 : {
12688 : /* Check if we have a usable scalar IV type for vectorization. */
12689 2670 : tree iters_vf_type = sizetype;
12690 2670 : if (!LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
12691 : {
12692 : /* Find the type with the minimum precision we can use
12693 : for the scalar IV. */
12694 2447 : tree cand_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
12695 :
12696 : /* Work out how many bits we need to represent the limit. */
12697 2447 : unsigned int min_ni_width
12698 2447 : = vect_min_prec_for_max_niters (loop_vinfo, 1);
12699 :
12700 : /* Check if we're using PFA, if so we need a signed IV and an
12701 : extra bit for the sign. */
12702 2447 : if (TYPE_UNSIGNED (cand_type)
12703 2447 : && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
12704 4018 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo))
12705 159 : min_ni_width += 1;
12706 :
12707 2447 : if (TYPE_PRECISION (cand_type) >= min_ni_width)
12708 2374 : iters_vf_type = unsigned_type_for (cand_type);
12709 : else
12710 : {
12711 73 : opt_scalar_int_mode cmp_mode_iter;
12712 73 : tree iv_type = NULL_TREE;
12713 357 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
12714 : {
12715 357 : auto cmp_mode = cmp_mode_iter.require ();
12716 357 : unsigned int cmp_bits = GET_MODE_BITSIZE (cmp_mode);
12717 357 : if (cmp_bits >= min_ni_width
12718 357 : && targetm.scalar_mode_supported_p (cmp_mode))
12719 : {
12720 73 : iv_type = build_nonstandard_integer_type (cmp_bits, true);
12721 73 : if (iv_type)
12722 : break;
12723 : }
12724 : }
12725 :
12726 73 : if (!iv_type)
12727 : {
12728 0 : if (dump_enabled_p ())
12729 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12730 : "can't vectorize early exit because the "
12731 : "target doesn't support a scalar type wide "
12732 : "wide enough to hold niters.\n");
12733 0 : return false;
12734 : }
12735 73 : iters_vf_type = iv_type;
12736 : }
12737 : }
12738 :
12739 2670 : LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo) = iters_vf_type;
12740 2670 : return true;
12741 : }
12742 :
12743 : /* Check to see if the current early break given in STMT_INFO is valid for
12744 : vectorization. */
12745 :
12746 : bool
12747 263593 : vectorizable_early_exit (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
12748 : gimple_stmt_iterator *gsi,
12749 : slp_tree slp_node, stmt_vector_for_cost *cost_vec)
12750 : {
12751 263593 : if (!is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
12752 : return false;
12753 :
12754 67662 : if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_condition_def)
12755 : return false;
12756 :
12757 67662 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
12758 : return false;
12759 :
12760 67662 : DUMP_VECT_SCOPE ("vectorizable_early_exit");
12761 :
12762 67662 : auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
12763 :
12764 : /* For SLP we don't want to use the type of the operands of the SLP node, when
12765 : vectorizing using SLP slp_node will be the children of the gcond and we
12766 : want to use the type of the direct children which since the gcond is root
12767 : will be the current node, rather than a child node as vect_is_simple_use
12768 : assumes. */
12769 67662 : tree vectype = SLP_TREE_VECTYPE (slp_node);
12770 67662 : if (!vectype)
12771 : return false;
12772 :
12773 67662 : machine_mode mode = TYPE_MODE (vectype);
12774 67662 : int vec_num = vect_get_num_copies (loop_vinfo, slp_node);
12775 :
12776 67662 : vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
12777 67662 : vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
12778 67662 : bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
12779 67662 : bool len_loop_p = LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
12780 :
12781 : /* Now build the new conditional. Pattern gimple_conds get dropped during
12782 : codegen so we must replace the original insn. */
12783 67662 : gimple *orig_stmt = STMT_VINFO_STMT (vect_orig_stmt (stmt_info));
12784 67662 : gcond *cond_stmt = as_a <gcond *>(orig_stmt);
12785 :
12786 67662 : tree vectype_out = vectype;
12787 67662 : auto bb = gimple_bb (cond_stmt);
12788 67662 : edge exit_true_edge = EDGE_SUCC (bb, 0);
12789 67662 : if (exit_true_edge->flags & EDGE_FALSE_VALUE)
12790 654 : exit_true_edge = EDGE_SUCC (bb, 1);
12791 67662 : gcc_assert (exit_true_edge->flags & EDGE_TRUE_VALUE);
12792 :
12793 : /* When vectorizing we assume that if the branch edge is taken that we're
12794 : exiting the loop. This is not however always the case as the compiler will
12795 : rewrite conditions to always be a comparison against 0. To do this it
12796 : sometimes flips the edges. This is fine for scalar, but for vector we
12797 : then have to negate the result of the test, as we're still assuming that if
12798 : you take the branch edge that we found the exit condition. i.e. we need to
12799 : know whether we are generating a `forall` or an `exist` condition. */
12800 135324 : bool flipped = flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo),
12801 67662 : exit_true_edge->dest);
12802 :
12803 : /* See if we support ADDHN and use that for the reduction. */
12804 67662 : internal_fn ifn = IFN_VEC_TRUNC_ADD_HIGH;
12805 67662 : bool addhn_supported_p
12806 67662 : = direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_BOTH);
12807 67662 : tree narrow_type = NULL_TREE;
12808 67662 : if (addhn_supported_p)
12809 : {
12810 : /* Calculate the narrowing type for the result. */
12811 0 : auto halfprec = TYPE_PRECISION (TREE_TYPE (vectype)) / 2;
12812 0 : auto unsignedp = TYPE_UNSIGNED (TREE_TYPE (vectype));
12813 0 : tree itype = build_nonstandard_integer_type (halfprec, unsignedp);
12814 0 : tree tmp_type = build_vector_type (itype, TYPE_VECTOR_SUBPARTS (vectype));
12815 0 : narrow_type = truth_type_for (tmp_type);
12816 :
12817 0 : if (!supports_vector_compare_and_branch (loop_vinfo,
12818 0 : TYPE_MODE (narrow_type)))
12819 : {
12820 0 : if (dump_enabled_p ())
12821 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12822 : "can't use ADDHN reduction because cbranch for "
12823 : "the narrowed type is not supported by the "
12824 : "target.\n");
12825 : addhn_supported_p = false;
12826 : }
12827 : }
12828 :
12829 : /* Analyze only. */
12830 67662 : if (cost_vec)
12831 : {
12832 66034 : if (!addhn_supported_p
12833 66034 : && !supports_vector_compare_and_branch (loop_vinfo, mode))
12834 : {
12835 63364 : if (dump_enabled_p ())
12836 597 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12837 : "can't vectorize early exit because the "
12838 : "target doesn't support flag setting vector "
12839 : "comparisons.\n");
12840 : return false;
12841 : }
12842 :
12843 2670 : if (!vectorizable_comparison_1 (loop_vinfo, vectype, stmt_info, code, gsi,
12844 : slp_node, cost_vec))
12845 : return false;
12846 :
12847 2670 : if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
12848 : {
12849 1571 : if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
12850 : OPTIMIZE_FOR_SPEED))
12851 0 : vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
12852 : else
12853 1571 : vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
12854 : }
12855 :
12856 2670 : if (!vect_compute_type_for_early_break_scalar_iv (loop_vinfo))
12857 : return false;
12858 :
12859 : return true;
12860 : }
12861 :
12862 : /* Transform. */
12863 :
12864 1628 : tree new_temp = NULL_TREE;
12865 1628 : gimple *new_stmt = NULL;
12866 :
12867 1628 : if (dump_enabled_p ())
12868 408 : dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
12869 :
12870 : /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
12871 : roots and so by the time we get to them we have already codegened the SLP tree
12872 : and so we shouldn't try to do so again. The arguments have already been
12873 : vectorized. It's not very clean to do this here, But the masking code below is
12874 : complex and this keeps it all in one place to ease fixes and backports. Once we
12875 : drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
12876 :
12877 1628 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
12878 1628 : basic_block cond_bb = gimple_bb (stmt);
12879 1628 : gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
12880 :
12881 1628 : auto_vec<tree> stmts;
12882 1628 : stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
12883 :
12884 : /* If we're comparing against a previous forall we need to negate the results
12885 : before we do the final comparison or reduction. */
12886 1628 : if (flipped)
12887 : {
12888 : /* Rewrite the if(all(mask)) into if (!all(mask)) which is the same as
12889 : if (any(~mask)) by negating the masks and flipping the branches.
12890 :
12891 : 1. For unmasked loops we simply reduce the ~mask.
12892 : 2. For masked loops we reduce (~mask & loop_mask) which is the same as
12893 : doing (mask & loop_mask) ^ loop_mask. */
12894 343 : for (unsigned i = 0; i < stmts.length (); i++)
12895 : {
12896 203 : tree inv_lhs = make_temp_ssa_name (vectype, NULL, "vexit_inv");
12897 203 : auto inv_stmt = gimple_build_assign (inv_lhs, BIT_NOT_EXPR, stmts[i]);
12898 203 : vect_finish_stmt_generation (loop_vinfo, stmt_info, inv_stmt,
12899 : &cond_gsi);
12900 203 : stmts[i] = inv_lhs;
12901 : }
12902 :
12903 140 : EDGE_SUCC (bb, 0)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12904 140 : EDGE_SUCC (bb, 1)->flags ^= (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE);
12905 : }
12906 :
12907 : /* Determine if we need to reduce the final value. */
12908 1628 : if (stmts.length () > 1)
12909 : {
12910 : /* We build the reductions in a way to maintain as much parallelism as
12911 : possible. */
12912 147 : auto_vec<tree> workset (stmts.length ());
12913 :
12914 : /* Mask the statements as we queue them up. Normally we loop over
12915 : vec_num, but since we inspect the exact results of vectorization
12916 : we don't need to and instead can just use the stmts themselves. */
12917 147 : if (masked_loop_p)
12918 0 : for (unsigned i = 0; i < stmts.length (); i++)
12919 : {
12920 0 : tree stmt_mask
12921 0 : = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
12922 : vectype, i);
12923 0 : stmt_mask
12924 0 : = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
12925 0 : stmts[i], &cond_gsi);
12926 0 : workset.quick_push (stmt_mask);
12927 : }
12928 147 : else if (len_loop_p)
12929 147 : for (unsigned i = 0; i < stmts.length (); i++)
12930 : {
12931 0 : tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
12932 : lens, vec_num,
12933 0 : vectype, stmts[i], i, 1);
12934 :
12935 0 : workset.quick_push (len_mask);
12936 : }
12937 : else
12938 147 : workset.splice (stmts);
12939 :
12940 442 : while (workset.length () > 1)
12941 : {
12942 295 : tree arg0 = workset.pop ();
12943 295 : tree arg1 = workset.pop ();
12944 295 : if (addhn_supported_p && workset.length () == 0)
12945 : {
12946 0 : new_stmt = gimple_build_call_internal (ifn, 2, arg0, arg1);
12947 0 : vectype_out = narrow_type;
12948 0 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12949 0 : gimple_call_set_lhs (as_a <gcall *> (new_stmt), new_temp);
12950 0 : gimple_call_set_nothrow (as_a <gcall *> (new_stmt), true);
12951 : }
12952 : else
12953 : {
12954 295 : new_temp = make_temp_ssa_name (vectype_out, NULL, "vexit_reduc");
12955 295 : new_stmt
12956 295 : = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
12957 : }
12958 295 : vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
12959 : &cond_gsi);
12960 295 : workset.quick_insert (0, new_temp);
12961 : }
12962 147 : }
12963 : else
12964 : {
12965 1481 : new_temp = stmts[0];
12966 1481 : if (masked_loop_p)
12967 : {
12968 2 : tree mask
12969 2 : = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
12970 2 : new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
12971 : new_temp, &cond_gsi);
12972 : }
12973 1479 : else if (len_loop_p)
12974 0 : new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
12975 : 1, vectype, new_temp, 0, 1);
12976 : }
12977 :
12978 1628 : gcc_assert (new_temp);
12979 :
12980 1628 : tree cst = build_zero_cst (vectype_out);
12981 1628 : gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
12982 1628 : update_stmt (orig_stmt);
12983 :
12984 : /* ??? */
12985 1628 : SLP_TREE_VEC_DEFS (slp_node).truncate (0);
12986 :
12987 1628 : return true;
12988 1628 : }
12989 :
12990 : /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
12991 : can handle all live statements in the node. Otherwise return true
12992 : if STMT_INFO is not live or if vectorizable_live_operation can handle it.
12993 : VEC_STMT_P is as for vectorizable_live_operation. */
12994 :
12995 : static bool
12996 1322112 : can_vectorize_live_stmts (vec_info *vinfo,
12997 : slp_tree slp_node, slp_instance slp_node_instance,
12998 : bool vec_stmt_p,
12999 : stmt_vector_for_cost *cost_vec)
13000 : {
13001 1322112 : stmt_vec_info slp_stmt_info;
13002 1322112 : unsigned int i;
13003 2787226 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
13004 : {
13005 1465114 : if (slp_stmt_info
13006 1448895 : && STMT_VINFO_LIVE_P (slp_stmt_info)
13007 1603184 : && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13008 : slp_node_instance, i,
13009 : vec_stmt_p, cost_vec))
13010 : return false;
13011 : }
13012 :
13013 : return true;
13014 : }
13015 :
13016 : /* Make sure the statement is vectorizable. */
13017 :
13018 : opt_result
13019 2639808 : vect_analyze_stmt (vec_info *vinfo,
13020 : slp_tree node, slp_instance node_instance,
13021 : stmt_vector_for_cost *cost_vec)
13022 : {
13023 2639808 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
13024 2639808 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
13025 2639808 : enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
13026 2639808 : bool ok;
13027 :
13028 2639808 : if (dump_enabled_p ())
13029 102783 : dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
13030 : stmt_info->stmt);
13031 :
13032 4981244 : if (gimple_has_volatile_ops (stmt_info->stmt))
13033 : {
13034 : /* ??? This shouldn't really happen, volatile stmts should
13035 : not end up in the SLP graph. */
13036 0 : return opt_result::failure_at (stmt_info->stmt,
13037 : "not vectorized:"
13038 : " stmt has volatile operands: %G\n",
13039 : stmt_info->stmt);
13040 : }
13041 :
13042 : /* Skip stmts that do not need to be vectorized. */
13043 2639808 : if (!STMT_VINFO_RELEVANT_P (stmt_info)
13044 0 : && !STMT_VINFO_LIVE_P (stmt_info))
13045 : {
13046 0 : if (dump_enabled_p ())
13047 0 : dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
13048 :
13049 : /* ??? This shouldn't really happen, irrelevant stmts should
13050 : not end up in the SLP graph. */
13051 0 : return opt_result::failure_at (stmt_info->stmt,
13052 : "not vectorized:"
13053 : " irrelevant stmt as SLP node %p "
13054 : "representative.\n",
13055 : (void *)node);
13056 : }
13057 :
13058 2639808 : switch (STMT_VINFO_DEF_TYPE (stmt_info))
13059 : {
13060 : case vect_internal_def:
13061 : case vect_condition_def:
13062 : break;
13063 :
13064 84757 : case vect_reduction_def:
13065 84757 : case vect_nested_cycle:
13066 84757 : gcc_assert (!bb_vinfo
13067 : && (relevance == vect_used_in_outer
13068 : || relevance == vect_used_in_outer_by_reduction
13069 : || relevance == vect_used_by_reduction
13070 : || relevance == vect_unused_in_scope
13071 : || relevance == vect_used_only_live));
13072 : break;
13073 :
13074 322 : case vect_double_reduction_def:
13075 322 : gcc_assert (!bb_vinfo && node);
13076 : break;
13077 :
13078 159806 : case vect_induction_def:
13079 159806 : case vect_first_order_recurrence:
13080 159806 : gcc_assert (!bb_vinfo);
13081 : break;
13082 :
13083 0 : case vect_constant_def:
13084 0 : case vect_external_def:
13085 0 : case vect_unknown_def_type:
13086 0 : default:
13087 0 : gcc_unreachable ();
13088 : }
13089 :
13090 2639808 : tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
13091 2639808 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13092 :
13093 2639808 : if (STMT_VINFO_RELEVANT_P (stmt_info))
13094 : {
13095 2639808 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
13096 2639808 : gcc_assert (SLP_TREE_VECTYPE (node)
13097 : || gimple_code (stmt_info->stmt) == GIMPLE_COND
13098 : || (call && gimple_call_lhs (call) == NULL_TREE));
13099 : }
13100 :
13101 2639808 : ok = true;
13102 2639808 : if (bb_vinfo
13103 1518043 : || (STMT_VINFO_RELEVANT_P (stmt_info)
13104 0 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
13105 : /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13106 : -mveclibabi= takes preference over library functions with
13107 : the simd attribute. */
13108 2639808 : ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
13109 2632640 : || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
13110 : cost_vec)
13111 2632163 : || vectorizable_conversion (vinfo, stmt_info, NULL, node, cost_vec)
13112 2546536 : || vectorizable_operation (vinfo, stmt_info, NULL, node, cost_vec)
13113 2122535 : || vectorizable_assignment (vinfo, stmt_info, NULL, node, cost_vec)
13114 2052126 : || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
13115 1593271 : || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
13116 762926 : || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
13117 718686 : || vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
13118 691664 : || vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
13119 545950 : || (bb_vinfo
13120 141724 : && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
13121 3125062 : || (is_a <loop_vec_info> (vinfo)
13122 404226 : && (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
13123 : stmt_info, node, cost_vec)
13124 403486 : || vectorizable_reduction (as_a <loop_vec_info> (vinfo),
13125 : stmt_info,
13126 : node, node_instance, cost_vec)
13127 321342 : || vectorizable_induction (as_a <loop_vec_info> (vinfo),
13128 : stmt_info, node, cost_vec)
13129 196991 : || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
13130 : stmt_info, node)
13131 196200 : || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13132 : stmt_info, node, cost_vec)
13133 195931 : || vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13134 : stmt_info, NULL, node,
13135 : cost_vec))));
13136 :
13137 2639808 : STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
13138 :
13139 2362849 : if (!ok)
13140 276959 : return opt_result::failure_at (stmt_info->stmt,
13141 : "not vectorized:"
13142 : " relevant stmt not supported: %G",
13143 : stmt_info->stmt);
13144 :
13145 : /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13146 : need extra handling, except for vectorizable reductions. */
13147 2362849 : if (!bb_vinfo
13148 1322112 : && (SLP_TREE_TYPE (node) != lc_phi_info_type
13149 791 : || SLP_TREE_DEF_TYPE (node) == vect_internal_def)
13150 1322112 : && (!node->ldst_lanes || SLP_TREE_PERMUTE_P (node))
13151 3684961 : && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
13152 : node, node_instance,
13153 : false, cost_vec))
13154 0 : return opt_result::failure_at (stmt_info->stmt,
13155 : "not vectorized:"
13156 : " live stmt not supported: %G",
13157 : stmt_info->stmt);
13158 :
13159 2362849 : return opt_result::success ();
13160 : }
13161 :
13162 :
13163 : /* Function vect_transform_stmt.
13164 :
13165 : Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13166 :
13167 : void
13168 998404 : vect_transform_stmt (vec_info *vinfo,
13169 : stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
13170 : slp_tree slp_node, slp_instance slp_node_instance)
13171 : {
13172 998404 : bool done;
13173 :
13174 998404 : gcc_assert (slp_node);
13175 :
13176 998404 : if (stmt_info)
13177 981415 : STMT_VINFO_VECTYPE (stmt_info) = NULL_TREE;
13178 :
13179 998404 : switch (SLP_TREE_TYPE (slp_node))
13180 : {
13181 23983 : case type_demotion_vec_info_type:
13182 23983 : case type_promotion_vec_info_type:
13183 23983 : case type_conversion_vec_info_type:
13184 23983 : done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
13185 23983 : gcc_assert (done);
13186 : break;
13187 :
13188 16428 : case induc_vec_info_type:
13189 16428 : done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
13190 : stmt_info, slp_node, NULL);
13191 16428 : gcc_assert (done);
13192 : break;
13193 :
13194 8968 : case shift_vec_info_type:
13195 8968 : done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
13196 8968 : gcc_assert (done);
13197 : break;
13198 :
13199 117658 : case op_vec_info_type:
13200 117658 : done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
13201 117658 : gcc_assert (done);
13202 : break;
13203 :
13204 16978 : case assignment_vec_info_type:
13205 16978 : done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
13206 16978 : gcc_assert (done);
13207 : break;
13208 :
13209 171668 : case load_vec_info_type:
13210 171668 : done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
13211 171668 : gcc_assert (done);
13212 : break;
13213 :
13214 557401 : case store_vec_info_type:
13215 557401 : done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
13216 557401 : gcc_assert (done);
13217 : break;
13218 :
13219 8674 : case condition_vec_info_type:
13220 8674 : done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
13221 8674 : gcc_assert (done);
13222 : break;
13223 :
13224 12687 : case comparison_vec_info_type:
13225 12687 : done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
13226 12687 : gcc_assert (done);
13227 : break;
13228 :
13229 4307 : case call_vec_info_type:
13230 4307 : done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
13231 4307 : break;
13232 :
13233 358 : case call_simd_clone_vec_info_type:
13234 358 : done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
13235 : slp_node, NULL);
13236 358 : break;
13237 :
13238 2699 : case reduc_vec_info_type:
13239 2699 : done = vect_transform_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
13240 : gsi, slp_node);
13241 2699 : gcc_assert (done);
13242 : break;
13243 :
13244 23769 : case cycle_phi_info_type:
13245 23769 : done = vect_transform_cycle_phi (as_a <loop_vec_info> (vinfo), stmt_info,
13246 : slp_node, slp_node_instance);
13247 23769 : gcc_assert (done);
13248 : break;
13249 :
13250 515 : case lc_phi_info_type:
13251 515 : done = vect_transform_lc_phi (as_a <loop_vec_info> (vinfo),
13252 : stmt_info, slp_node);
13253 515 : gcc_assert (done);
13254 : break;
13255 :
13256 43 : case recurr_info_type:
13257 43 : done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
13258 : stmt_info, slp_node, NULL);
13259 43 : gcc_assert (done);
13260 : break;
13261 :
13262 15279 : case phi_info_type:
13263 15279 : done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
13264 : stmt_info, slp_node, NULL);
13265 15279 : gcc_assert (done);
13266 : break;
13267 :
13268 0 : case loop_exit_ctrl_vec_info_type:
13269 0 : done = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
13270 : stmt_info, gsi, slp_node, NULL);
13271 0 : gcc_assert (done);
13272 : break;
13273 :
13274 16989 : case permute_info_type:
13275 16989 : done = vectorizable_slp_permutation (vinfo, gsi, slp_node, NULL);
13276 16989 : gcc_assert (done);
13277 : break;
13278 :
13279 0 : default:
13280 0 : if (!STMT_VINFO_LIVE_P (stmt_info))
13281 : {
13282 0 : if (dump_enabled_p ())
13283 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13284 : "stmt not supported.\n");
13285 0 : gcc_unreachable ();
13286 : }
13287 998404 : done = true;
13288 : }
13289 :
13290 998404 : if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
13291 441003 : && (!slp_node->ldst_lanes || SLP_TREE_PERMUTE_P (slp_node)))
13292 : {
13293 : /* Handle stmts whose DEF is used outside the loop-nest that is
13294 : being vectorized. */
13295 615081 : for (unsigned lane : SLP_TREE_LIVE_LANES (slp_node))
13296 : {
13297 75368 : stmt_vec_info slp_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[lane];
13298 75368 : done = vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
13299 : slp_node_instance, lane,
13300 : true, NULL);
13301 75368 : gcc_assert (done);
13302 : }
13303 : }
13304 998404 : }
13305 :
13306 :
13307 : /* Remove a group of stores (for SLP or interleaving), free their
13308 : stmt_vec_info. */
13309 :
13310 : void
13311 0 : vect_remove_stores (vec_info *vinfo, stmt_vec_info first_stmt_info)
13312 : {
13313 0 : stmt_vec_info next_stmt_info = first_stmt_info;
13314 :
13315 0 : while (next_stmt_info)
13316 : {
13317 0 : stmt_vec_info tmp = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
13318 0 : next_stmt_info = vect_orig_stmt (next_stmt_info);
13319 : /* Free the attached stmt_vec_info and remove the stmt. */
13320 0 : vinfo->remove_stmt (next_stmt_info);
13321 0 : next_stmt_info = tmp;
13322 : }
13323 0 : }
13324 :
13325 : /* If NUNITS is nonzero, return a vector type that contains NUNITS
13326 : elements of type SCALAR_TYPE, or null if the target doesn't support
13327 : such a type.
13328 :
13329 : If NUNITS is zero, return a vector type that contains elements of
13330 : type SCALAR_TYPE, choosing whichever vector size the target prefers.
13331 :
13332 : If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13333 : for this vectorization region and want to "autodetect" the best choice.
13334 : Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13335 : and we want the new type to be interoperable with it. PREVAILING_MODE
13336 : in this case can be a scalar integer mode or a vector mode; when it
13337 : is a vector mode, the function acts like a tree-level version of
13338 : related_vector_mode. */
13339 :
13340 : tree
13341 30842203 : get_related_vectype_for_scalar_type (machine_mode prevailing_mode,
13342 : tree scalar_type, poly_uint64 nunits)
13343 : {
13344 30842203 : tree orig_scalar_type = scalar_type;
13345 30842203 : scalar_mode inner_mode;
13346 30842203 : machine_mode simd_mode;
13347 30842203 : tree vectype;
13348 :
13349 30842203 : if ((!INTEGRAL_TYPE_P (scalar_type)
13350 10065087 : && !POINTER_TYPE_P (scalar_type)
13351 2136503 : && !SCALAR_FLOAT_TYPE_P (scalar_type))
13352 40374580 : || (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
13353 1603880 : && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode)))
13354 : return NULL_TREE;
13355 :
13356 30306184 : unsigned int nbytes = GET_MODE_SIZE (inner_mode);
13357 :
13358 : /* Interoperability between modes requires one to be a constant multiple
13359 : of the other, so that the number of vectors required for each operation
13360 : is a compile-time constant. */
13361 30306184 : if (prevailing_mode != VOIDmode
13362 29137779 : && !constant_multiple_p (nunits * nbytes,
13363 29137779 : GET_MODE_SIZE (prevailing_mode))
13364 31539859 : && !constant_multiple_p (GET_MODE_SIZE (prevailing_mode),
13365 1233675 : nunits * nbytes))
13366 : return NULL_TREE;
13367 :
13368 : /* For vector types of elements whose mode precision doesn't
13369 : match their types precision we use a element type of mode
13370 : precision. The vectorization routines will have to make sure
13371 : they support the proper result truncation/extension.
13372 : We also make sure to build vector types with INTEGER_TYPE
13373 : component type only. */
13374 30306184 : if (INTEGRAL_TYPE_P (scalar_type)
13375 51083213 : && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
13376 19578678 : || TREE_CODE (scalar_type) != INTEGER_TYPE))
13377 1415666 : scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
13378 1415666 : TYPE_UNSIGNED (scalar_type));
13379 :
13380 : /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13381 : When the component mode passes the above test simply use a type
13382 : corresponding to that mode. The theory is that any use that
13383 : would cause problems with this will disable vectorization anyway. */
13384 28890518 : else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
13385 : && !INTEGRAL_TYPE_P (scalar_type))
13386 7928584 : scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
13387 :
13388 : /* We can't build a vector type of elements with alignment bigger than
13389 : their size. */
13390 20961934 : else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
13391 411498 : scalar_type = lang_hooks.types.type_for_mode (inner_mode,
13392 205749 : TYPE_UNSIGNED (scalar_type));
13393 :
13394 : /* If we felt back to using the mode fail if there was
13395 : no scalar type for it. */
13396 30306184 : if (scalar_type == NULL_TREE)
13397 : return NULL_TREE;
13398 :
13399 : /* If no prevailing mode was supplied, use the mode the target prefers.
13400 : Otherwise lookup a vector mode based on the prevailing mode. */
13401 30306184 : if (prevailing_mode == VOIDmode)
13402 : {
13403 1168405 : gcc_assert (known_eq (nunits, 0U));
13404 1168405 : simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
13405 1168405 : if (SCALAR_INT_MODE_P (simd_mode))
13406 : {
13407 : /* Traditional behavior is not to take the integer mode
13408 : literally, but simply to use it as a way of determining
13409 : the vector size. It is up to mode_for_vector to decide
13410 : what the TYPE_MODE should be.
13411 :
13412 : Note that nunits == 1 is allowed in order to support single
13413 : element vector types. */
13414 65596 : if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits)
13415 32798 : || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13416 : return NULL_TREE;
13417 : }
13418 : }
13419 29137779 : else if (SCALAR_INT_MODE_P (prevailing_mode)
13420 29137779 : || !related_vector_mode (prevailing_mode,
13421 29137141 : inner_mode, nunits).exists (&simd_mode))
13422 : {
13423 : /* Fall back to using mode_for_vector, mostly in the hope of being
13424 : able to use an integer mode. */
13425 2276986 : if (known_eq (nunits, 0U)
13426 4548393 : && !multiple_p (GET_MODE_SIZE (prevailing_mode), nbytes, &nunits))
13427 : return NULL_TREE;
13428 :
13429 238056 : if (!mode_for_vector (inner_mode, nunits).exists (&simd_mode))
13430 223993 : return NULL_TREE;
13431 : }
13432 :
13433 28011113 : vectype = build_vector_type_for_mode (scalar_type, simd_mode);
13434 :
13435 : /* In cases where the mode was chosen by mode_for_vector, check that
13436 : the target actually supports the chosen mode, or that it at least
13437 : allows the vector mode to be replaced by a like-sized integer. */
13438 56022226 : if (!VECTOR_MODE_P (TYPE_MODE (vectype))
13439 28025475 : && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
13440 : return NULL_TREE;
13441 :
13442 : /* Re-attach the address-space qualifier if we canonicalized the scalar
13443 : type. */
13444 28002898 : if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
13445 3 : return build_qualified_type
13446 3 : (vectype, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type)));
13447 :
13448 : return vectype;
13449 : }
13450 :
13451 : /* Function get_vectype_for_scalar_type.
13452 :
13453 : Returns the vector type corresponding to SCALAR_TYPE as supported
13454 : by the target. If GROUP_SIZE is nonzero and we're performing BB
13455 : vectorization, make sure that the number of elements in the vector
13456 : is no bigger than GROUP_SIZE. */
13457 :
13458 : tree
13459 26735117 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type,
13460 : unsigned int group_size)
13461 : {
13462 : /* For BB vectorization, we should always have a group size once we've
13463 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13464 : are tentative requests during things like early data reference
13465 : analysis and pattern recognition. */
13466 26735117 : if (is_a <bb_vec_info> (vinfo))
13467 23613137 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
13468 : else
13469 : group_size = 0;
13470 :
13471 26735117 : tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13472 : scalar_type);
13473 26735117 : if (vectype && vinfo->vector_mode == VOIDmode)
13474 1084571 : vinfo->vector_mode = TYPE_MODE (vectype);
13475 :
13476 : /* Register the natural choice of vector type, before the group size
13477 : has been applied. */
13478 0 : if (vectype)
13479 24068241 : vinfo->used_vector_modes.add (TYPE_MODE (vectype));
13480 :
13481 : /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13482 : try again with an explicit number of elements. */
13483 24068241 : if (vectype
13484 24068241 : && group_size
13485 26735117 : && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size))
13486 : {
13487 : /* Start with the biggest number of units that fits within
13488 : GROUP_SIZE and halve it until we find a valid vector type.
13489 : Usually either the first attempt will succeed or all will
13490 : fail (in the latter case because GROUP_SIZE is too small
13491 : for the target), but it's possible that a target could have
13492 : a hole between supported vector types.
13493 :
13494 : If GROUP_SIZE is not a power of 2, this has the effect of
13495 : trying the largest power of 2 that fits within the group,
13496 : even though the group is not a multiple of that vector size.
13497 : The BB vectorizer will then try to carve up the group into
13498 : smaller pieces. */
13499 2572999 : unsigned int nunits = 1 << floor_log2 (group_size);
13500 2572999 : do
13501 : {
13502 2572999 : vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode,
13503 2572999 : scalar_type, nunits);
13504 2572999 : nunits /= 2;
13505 : }
13506 2572999 : while (nunits > 1 && !vectype);
13507 : }
13508 :
13509 26735117 : return vectype;
13510 : }
13511 :
13512 : /* Return the vector type corresponding to SCALAR_TYPE as supported
13513 : by the target. NODE, if nonnull, is the SLP tree node that will
13514 : use the returned vector type. */
13515 :
13516 : tree
13517 181893 : get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node)
13518 : {
13519 181893 : unsigned int group_size = 0;
13520 181893 : if (node)
13521 181893 : group_size = SLP_TREE_LANES (node);
13522 181893 : return get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13523 : }
13524 :
13525 : /* Function get_mask_type_for_scalar_type.
13526 :
13527 : Returns the mask type corresponding to a result of comparison
13528 : of vectors of specified SCALAR_TYPE as supported by target.
13529 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
13530 : make sure that the number of elements in the vector is no bigger
13531 : than GROUP_SIZE. */
13532 :
13533 : tree
13534 1207960 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13535 : unsigned int group_size)
13536 : {
13537 1207960 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
13538 :
13539 1207960 : if (!vectype)
13540 : return NULL;
13541 :
13542 1187711 : return truth_type_for (vectype);
13543 : }
13544 :
13545 : /* Function get_mask_type_for_scalar_type.
13546 :
13547 : Returns the mask type corresponding to a result of comparison
13548 : of vectors of specified SCALAR_TYPE as supported by target.
13549 : NODE, if nonnull, is the SLP tree node that will use the returned
13550 : vector type. */
13551 :
13552 : tree
13553 19 : get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
13554 : slp_tree node)
13555 : {
13556 19 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
13557 :
13558 19 : if (!vectype)
13559 : return NULL;
13560 :
13561 19 : return truth_type_for (vectype);
13562 : }
13563 :
13564 : /* Function get_same_sized_vectype
13565 :
13566 : Returns a vector type corresponding to SCALAR_TYPE of size
13567 : VECTOR_TYPE if supported by the target. */
13568 :
13569 : tree
13570 167328 : get_same_sized_vectype (tree scalar_type, tree vector_type)
13571 : {
13572 167328 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
13573 0 : return truth_type_for (vector_type);
13574 :
13575 167328 : poly_uint64 nunits;
13576 334656 : if (!multiple_p (GET_MODE_SIZE (TYPE_MODE (vector_type)),
13577 334656 : GET_MODE_SIZE (TYPE_MODE (scalar_type)), &nunits))
13578 : return NULL_TREE;
13579 :
13580 167328 : return get_related_vectype_for_scalar_type (TYPE_MODE (vector_type),
13581 167328 : scalar_type, nunits);
13582 : }
13583 :
13584 : /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13585 : would not change the chosen vector modes. */
13586 :
13587 : bool
13588 1655683 : vect_chooses_same_modes_p (vec_info *vinfo, machine_mode vector_mode)
13589 : {
13590 1655683 : for (vec_info::mode_set::iterator i = vinfo->used_vector_modes.begin ();
13591 2723896 : i != vinfo->used_vector_modes.end (); ++i)
13592 1951661 : if (!VECTOR_MODE_P (*i)
13593 5854713 : || related_vector_mode (vector_mode, GET_MODE_INNER (*i), 0) != *i)
13594 883448 : return false;
13595 772235 : return true;
13596 : }
13597 :
13598 : /* Return true if replacing VECTOR_MODE with ALT_VECTOR_MODE would not
13599 : change the chosen vector modes for analysis of a loop. */
13600 :
13601 : bool
13602 393865 : vect_chooses_same_modes_p (machine_mode vector_mode,
13603 : machine_mode alt_vector_mode)
13604 : {
13605 64330 : return (VECTOR_MODE_P (vector_mode)
13606 393865 : && VECTOR_MODE_P (alt_vector_mode)
13607 787730 : && (related_vector_mode (vector_mode,
13608 : GET_MODE_INNER (alt_vector_mode))
13609 393865 : == alt_vector_mode)
13610 420435 : && (related_vector_mode (alt_vector_mode,
13611 : GET_MODE_INNER (vector_mode))
13612 13285 : == vector_mode));
13613 : }
13614 :
13615 : /* Function vect_is_simple_use.
13616 :
13617 : Input:
13618 : VINFO - the vect info of the loop or basic block that is being vectorized.
13619 : OPERAND - operand in the loop or bb.
13620 : Output:
13621 : DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13622 : case OPERAND is an SSA_NAME that is defined in the vectorizable region
13623 : DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13624 : the definition could be anywhere in the function
13625 : DT - the type of definition
13626 :
13627 : Returns whether a stmt with OPERAND can be vectorized.
13628 : For loops, supportable operands are constants, loop invariants, and operands
13629 : that are defined by the current iteration of the loop. Unsupportable
13630 : operands are those that are defined by a previous iteration of the loop (as
13631 : is the case in reduction/induction computations).
13632 : For basic blocks, supportable operands are constants and bb invariants.
13633 : For now, operands defined outside the basic block are not supported. */
13634 :
13635 : bool
13636 44080551 : vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
13637 : stmt_vec_info *def_stmt_info_out, gimple **def_stmt_out)
13638 : {
13639 44080551 : if (def_stmt_info_out)
13640 42045583 : *def_stmt_info_out = NULL;
13641 44080551 : if (def_stmt_out)
13642 10088768 : *def_stmt_out = NULL;
13643 44080551 : *dt = vect_unknown_def_type;
13644 :
13645 44080551 : if (dump_enabled_p ())
13646 : {
13647 786870 : dump_printf_loc (MSG_NOTE, vect_location,
13648 : "vect_is_simple_use: operand ");
13649 786870 : if (TREE_CODE (operand) == SSA_NAME
13650 786870 : && !SSA_NAME_IS_DEFAULT_DEF (operand))
13651 720181 : dump_gimple_expr (MSG_NOTE, TDF_SLIM, SSA_NAME_DEF_STMT (operand), 0);
13652 : else
13653 66689 : dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
13654 : }
13655 :
13656 44080551 : if (CONSTANT_CLASS_P (operand))
13657 3352194 : *dt = vect_constant_def;
13658 40728357 : else if (is_gimple_min_invariant (operand))
13659 398668 : *dt = vect_external_def;
13660 40329689 : else if (TREE_CODE (operand) != SSA_NAME)
13661 1038 : *dt = vect_unknown_def_type;
13662 40328651 : else if (SSA_NAME_IS_DEFAULT_DEF (operand))
13663 695181 : *dt = vect_external_def;
13664 : else
13665 : {
13666 39633470 : gimple *def_stmt = SSA_NAME_DEF_STMT (operand);
13667 39633470 : stmt_vec_info stmt_vinfo = vinfo->lookup_def (operand);
13668 39633470 : if (!stmt_vinfo)
13669 888737 : *dt = vect_external_def;
13670 : else
13671 : {
13672 38744733 : stmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
13673 38744733 : def_stmt = stmt_vinfo->stmt;
13674 38744733 : *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
13675 38744733 : if (def_stmt_info_out)
13676 36718549 : *def_stmt_info_out = stmt_vinfo;
13677 : }
13678 39633470 : if (def_stmt_out)
13679 9868305 : *def_stmt_out = def_stmt;
13680 : }
13681 :
13682 44080551 : if (dump_enabled_p ())
13683 : {
13684 786870 : dump_printf (MSG_NOTE, ", type of def: ");
13685 786870 : switch (*dt)
13686 : {
13687 0 : case vect_uninitialized_def:
13688 0 : dump_printf (MSG_NOTE, "uninitialized\n");
13689 0 : break;
13690 55498 : case vect_constant_def:
13691 55498 : dump_printf (MSG_NOTE, "constant\n");
13692 55498 : break;
13693 27261 : case vect_external_def:
13694 27261 : dump_printf (MSG_NOTE, "external\n");
13695 27261 : break;
13696 562186 : case vect_internal_def:
13697 562186 : dump_printf (MSG_NOTE, "internal\n");
13698 562186 : break;
13699 110055 : case vect_induction_def:
13700 110055 : dump_printf (MSG_NOTE, "induction\n");
13701 110055 : break;
13702 28485 : case vect_reduction_def:
13703 28485 : dump_printf (MSG_NOTE, "reduction\n");
13704 28485 : break;
13705 482 : case vect_double_reduction_def:
13706 482 : dump_printf (MSG_NOTE, "double reduction\n");
13707 482 : break;
13708 2178 : case vect_nested_cycle:
13709 2178 : dump_printf (MSG_NOTE, "nested cycle\n");
13710 2178 : break;
13711 277 : case vect_first_order_recurrence:
13712 277 : dump_printf (MSG_NOTE, "first order recurrence\n");
13713 277 : break;
13714 0 : case vect_condition_def:
13715 0 : dump_printf (MSG_NOTE, "control flow\n");
13716 0 : break;
13717 448 : case vect_unknown_def_type:
13718 448 : dump_printf (MSG_NOTE, "unknown\n");
13719 448 : break;
13720 : }
13721 : }
13722 :
13723 44080551 : if (*dt == vect_unknown_def_type)
13724 : {
13725 55680 : if (dump_enabled_p ())
13726 448 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13727 : "Unsupported pattern.\n");
13728 : return false;
13729 : }
13730 :
13731 : return true;
13732 : }
13733 :
13734 : /* Function vect_is_simple_use.
13735 :
13736 : Same as vect_is_simple_use but determines the operand by operand
13737 : position OPERAND from either STMT or SLP_NODE, filling in *OP
13738 : and *SLP_DEF (when SLP_NODE is not NULL). */
13739 :
13740 : bool
13741 3712144 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13742 : unsigned operand, tree *op, slp_tree *slp_def,
13743 : enum vect_def_type *dt,
13744 : tree *vectype, stmt_vec_info *def_stmt_info_out)
13745 : {
13746 3712144 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
13747 3712144 : *slp_def = child;
13748 3712144 : *vectype = SLP_TREE_VECTYPE (child);
13749 3712144 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
13750 : {
13751 : /* ??? VEC_PERM nodes might be intermediate and their lane value
13752 : have no representative (nor do we build a VEC_PERM stmt for
13753 : the actual operation). Note for two-operator nodes we set
13754 : a representative but leave scalar stmts empty as we'd only
13755 : have one for a subset of lanes. Ideally no caller would
13756 : require *op for internal defs. */
13757 2025324 : if (SLP_TREE_REPRESENTATIVE (child))
13758 : {
13759 1967839 : *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
13760 1967839 : return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
13761 : }
13762 : else
13763 : {
13764 57485 : gcc_assert (SLP_TREE_PERMUTE_P (child));
13765 57485 : *op = error_mark_node;
13766 57485 : *dt = vect_internal_def;
13767 57485 : if (def_stmt_info_out)
13768 149 : *def_stmt_info_out = NULL;
13769 : return true;
13770 : }
13771 : }
13772 : else
13773 : {
13774 1686820 : if (def_stmt_info_out)
13775 50580 : *def_stmt_info_out = NULL;
13776 1686820 : *op = SLP_TREE_SCALAR_OPS (child)[0];
13777 1686820 : *dt = SLP_TREE_DEF_TYPE (child);
13778 1686820 : return true;
13779 : }
13780 : }
13781 :
13782 : /* Wrapper around vect_is_simple_use that elides the scalar op output. */
13783 :
13784 : bool
13785 278864 : vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
13786 : unsigned operand, slp_tree *slp_def,
13787 : enum vect_def_type *dt, tree *vectype)
13788 : {
13789 278864 : tree op;
13790 278864 : return vect_is_simple_use (vinfo, slp_node, operand, &op, slp_def, dt,
13791 278864 : vectype);
13792 : }
13793 :
13794 : /* If OP is not NULL and is external or constant update its vector
13795 : type with VECTYPE. Returns true if successful or false if not,
13796 : for example when conflicting vector types are present. */
13797 :
13798 : bool
13799 3223748 : vect_maybe_update_slp_op_vectype (slp_tree op, tree vectype)
13800 : {
13801 3223748 : if (!op || SLP_TREE_DEF_TYPE (op) == vect_internal_def)
13802 : return true;
13803 1112474 : if (SLP_TREE_VECTYPE (op))
13804 23085 : return types_compatible_p (SLP_TREE_VECTYPE (op), vectype);
13805 : /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13806 : should be handled by patters. Allow vect_constant_def for now
13807 : as well as the trivial single-lane uniform vect_external_def case
13808 : both of which we code-generate reasonably. */
13809 1089389 : if (VECTOR_BOOLEAN_TYPE_P (vectype)
13810 1725 : && SLP_TREE_DEF_TYPE (op) == vect_external_def
13811 1090647 : && SLP_TREE_LANES (op) > 1)
13812 : return false;
13813 1089187 : SLP_TREE_VECTYPE (op) = vectype;
13814 1089187 : return true;
13815 : }
13816 :
13817 : /* Function supportable_widening_operation
13818 :
13819 : Check whether an operation represented by the code CODE is a
13820 : widening operation that is supported by the target platform in
13821 : vector form (i.e., when operating on arguments of type VECTYPE_IN
13822 : producing a result of type VECTYPE_OUT).
13823 :
13824 : Widening operations we currently support are NOP (CONVERT), FLOAT,
13825 : FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13826 : are supported by the target platform either directly (via vector
13827 : tree-codes), or via target builtins.
13828 :
13829 : When EVENODD_OK then also lane-swizzling operations are considered.
13830 :
13831 : Output:
13832 : - CODE1 and CODE2 are codes of vector operations to be used when
13833 : vectorizing the operation, if available.
13834 : - MULTI_STEP_CVT determines the number of required intermediate steps in
13835 : case of multi-step conversion (like char->short->int - in that case
13836 : MULTI_STEP_CVT will be 1).
13837 : - INTERM_TYPES contains the intermediate type required to perform the
13838 : widening operation (short in the above example). */
13839 :
13840 : bool
13841 525567 : supportable_widening_operation (code_helper code,
13842 : tree vectype_out, tree vectype_in,
13843 : bool evenodd_ok,
13844 : code_helper *code1,
13845 : code_helper *code2,
13846 : int *multi_step_cvt,
13847 : vec<tree> *interm_types)
13848 : {
13849 525567 : machine_mode vec_mode;
13850 525567 : enum insn_code icode1, icode2;
13851 525567 : optab optab1 = unknown_optab, optab2 = unknown_optab;
13852 525567 : tree vectype = vectype_in;
13853 525567 : tree wide_vectype = vectype_out;
13854 525567 : tree_code c1 = MAX_TREE_CODES, c2 = MAX_TREE_CODES;
13855 525567 : int i;
13856 525567 : tree prev_type, intermediate_type;
13857 525567 : machine_mode intermediate_mode, prev_mode;
13858 525567 : optab optab3, optab4;
13859 :
13860 525567 : *multi_step_cvt = 0;
13861 :
13862 525567 : switch (code.safe_as_tree_code ())
13863 : {
13864 : case MAX_TREE_CODES:
13865 : /* Don't set c1 and c2 if code is not a tree_code. */
13866 : break;
13867 :
13868 205836 : case WIDEN_MULT_EXPR:
13869 : /* The result of a vectorized widening operation usually requires
13870 : two vectors (because the widened results do not fit into one vector).
13871 : The generated vector results would normally be expected to be
13872 : generated in the same order as in the original scalar computation,
13873 : i.e. if 8 results are generated in each vector iteration, they are
13874 : to be organized as follows:
13875 : vect1: [res1,res2,res3,res4],
13876 : vect2: [res5,res6,res7,res8].
13877 :
13878 : However, in the special case that the result of the widening
13879 : operation is used in a reduction computation only, the order doesn't
13880 : matter (because when vectorizing a reduction we change the order of
13881 : the computation). Some targets can take advantage of this and
13882 : generate more efficient code. For example, targets like Altivec,
13883 : that support widen_mult using a sequence of {mult_even,mult_odd}
13884 : generate the following vectors:
13885 : vect1: [res1,res3,res5,res7],
13886 : vect2: [res2,res4,res6,res8].
13887 :
13888 : When vectorizing outer-loops, we execute the inner-loop sequentially
13889 : (each vectorized inner-loop iteration contributes to VF outer-loop
13890 : iterations in parallel). We therefore don't allow to change the
13891 : order of the computation in the inner-loop during outer-loop
13892 : vectorization. */
13893 : /* TODO: Another case in which order doesn't *really* matter is when we
13894 : widen and then contract again, e.g. (short)((int)x * y >> 8).
13895 : Normally, pack_trunc performs an even/odd permute, whereas the
13896 : repack from an even/odd expansion would be an interleave, which
13897 : would be significantly simpler for e.g. AVX2. */
13898 : /* In any case, in order to avoid duplicating the code below, recurse
13899 : on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13900 : are properly set up for the caller. If we fail, we'll continue with
13901 : a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13902 205836 : if (evenodd_ok
13903 205836 : && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
13904 : vectype_out, vectype_in,
13905 : evenodd_ok, code1,
13906 : code2, multi_step_cvt,
13907 : interm_types))
13908 106337 : return true;
13909 : c1 = VEC_WIDEN_MULT_LO_EXPR;
13910 : c2 = VEC_WIDEN_MULT_HI_EXPR;
13911 : break;
13912 :
13913 : case DOT_PROD_EXPR:
13914 419230 : c1 = DOT_PROD_EXPR;
13915 419230 : c2 = DOT_PROD_EXPR;
13916 : break;
13917 :
13918 0 : case SAD_EXPR:
13919 0 : c1 = SAD_EXPR;
13920 0 : c2 = SAD_EXPR;
13921 0 : break;
13922 :
13923 203761 : case VEC_WIDEN_MULT_EVEN_EXPR:
13924 : /* Support the recursion induced just above. */
13925 203761 : c1 = VEC_WIDEN_MULT_EVEN_EXPR;
13926 203761 : c2 = VEC_WIDEN_MULT_ODD_EXPR;
13927 203761 : break;
13928 :
13929 9666 : case WIDEN_LSHIFT_EXPR:
13930 9666 : c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
13931 9666 : c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
13932 9666 : break;
13933 :
13934 45257 : CASE_CONVERT:
13935 45257 : c1 = VEC_UNPACK_LO_EXPR;
13936 45257 : c2 = VEC_UNPACK_HI_EXPR;
13937 45257 : break;
13938 :
13939 9216 : case FLOAT_EXPR:
13940 9216 : c1 = VEC_UNPACK_FLOAT_LO_EXPR;
13941 9216 : c2 = VEC_UNPACK_FLOAT_HI_EXPR;
13942 9216 : break;
13943 :
13944 127 : case FIX_TRUNC_EXPR:
13945 127 : c1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
13946 127 : c2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
13947 127 : break;
13948 :
13949 0 : default:
13950 0 : gcc_unreachable ();
13951 : }
13952 :
13953 419230 : if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
13954 : std::swap (c1, c2);
13955 :
13956 419230 : if (code == FIX_TRUNC_EXPR)
13957 : {
13958 : /* The signedness is determined from output operand. */
13959 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
13960 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
13961 : }
13962 749618 : else if (CONVERT_EXPR_CODE_P (code.safe_as_tree_code ())
13963 45257 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
13964 7995 : && VECTOR_BOOLEAN_TYPE_P (vectype)
13965 7995 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
13966 367895 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
13967 : {
13968 : /* If the input and result modes are the same, a different optab
13969 : is needed where we pass in the number of units in vectype. */
13970 : optab1 = vec_unpacks_sbool_lo_optab;
13971 : optab2 = vec_unpacks_sbool_hi_optab;
13972 : }
13973 :
13974 419230 : vec_mode = TYPE_MODE (vectype);
13975 419230 : if (widening_fn_p (code))
13976 : {
13977 : /* If this is an internal fn then we must check whether the target
13978 : supports either a low-high split or an even-odd split. */
13979 51704 : internal_fn ifn = as_internal_fn ((combined_fn) code);
13980 :
13981 51704 : internal_fn lo, hi, even, odd;
13982 51704 : lookup_hilo_internal_fn (ifn, &lo, &hi);
13983 51704 : if (BYTES_BIG_ENDIAN)
13984 : std::swap (lo, hi);
13985 51704 : *code1 = as_combined_fn (lo);
13986 51704 : *code2 = as_combined_fn (hi);
13987 51704 : optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
13988 51704 : optab2 = direct_internal_fn_optab (hi, {vectype, vectype});
13989 :
13990 : /* If we don't support low-high, then check for even-odd. */
13991 51704 : if (!optab1
13992 51704 : || (icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
13993 0 : || !optab2
13994 51704 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
13995 : {
13996 51704 : lookup_evenodd_internal_fn (ifn, &even, &odd);
13997 51704 : *code1 = as_combined_fn (even);
13998 51704 : *code2 = as_combined_fn (odd);
13999 51704 : optab1 = direct_internal_fn_optab (even, {vectype, vectype});
14000 51704 : optab2 = direct_internal_fn_optab (odd, {vectype, vectype});
14001 : }
14002 : }
14003 367526 : else if (code.is_tree_code ())
14004 : {
14005 367526 : if (code == FIX_TRUNC_EXPR)
14006 : {
14007 : /* The signedness is determined from output operand. */
14008 127 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14009 127 : optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
14010 : }
14011 367399 : else if (CONVERT_EXPR_CODE_P ((tree_code) code.safe_as_tree_code ())
14012 45257 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14013 7995 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14014 7995 : && TYPE_MODE (wide_vectype) == TYPE_MODE (vectype)
14015 367895 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype)))
14016 : {
14017 : /* If the input and result modes are the same, a different optab
14018 : is needed where we pass in the number of units in vectype. */
14019 : optab1 = vec_unpacks_sbool_lo_optab;
14020 : optab2 = vec_unpacks_sbool_hi_optab;
14021 : }
14022 : else
14023 : {
14024 366903 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14025 366903 : optab2 = optab_for_tree_code (c2, vectype, optab_default);
14026 : }
14027 367526 : *code1 = c1;
14028 367526 : *code2 = c2;
14029 : }
14030 :
14031 419230 : if (!optab1 || !optab2)
14032 : return false;
14033 :
14034 419230 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
14035 419230 : || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
14036 : return false;
14037 :
14038 :
14039 165460 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14040 165460 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14041 : {
14042 153738 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14043 : return true;
14044 : /* For scalar masks we may have different boolean
14045 : vector types having the same QImode. Thus we
14046 : add additional check for elements number. */
14047 4188 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
14048 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14049 : return true;
14050 : }
14051 :
14052 : /* Check if it's a multi-step conversion that can be done using intermediate
14053 : types. */
14054 :
14055 11951 : prev_type = vectype;
14056 11951 : prev_mode = vec_mode;
14057 :
14058 11951 : if (!CONVERT_EXPR_CODE_P (code.safe_as_tree_code ()))
14059 : return false;
14060 :
14061 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14062 : intermediate steps in promotion sequence. We try
14063 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14064 : not. */
14065 11899 : interm_types->create (MAX_INTERM_CVT_STEPS);
14066 25129 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14067 : {
14068 13230 : intermediate_mode = insn_data[icode1].operand[0].mode;
14069 13230 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14070 4713 : intermediate_type
14071 4713 : = vect_halve_mask_nunits (prev_type, intermediate_mode);
14072 8517 : else if (VECTOR_MODE_P (intermediate_mode))
14073 : {
14074 8517 : tree intermediate_element_type
14075 8517 : = lang_hooks.types.type_for_mode (GET_MODE_INNER (intermediate_mode),
14076 8517 : TYPE_UNSIGNED (prev_type));
14077 8517 : intermediate_type
14078 8517 : = build_vector_type_for_mode (intermediate_element_type,
14079 : intermediate_mode);
14080 8517 : }
14081 : else
14082 0 : intermediate_type
14083 0 : = lang_hooks.types.type_for_mode (intermediate_mode,
14084 0 : TYPE_UNSIGNED (prev_type));
14085 :
14086 13230 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14087 4713 : && VECTOR_BOOLEAN_TYPE_P (wide_vectype)
14088 4713 : && intermediate_mode == TYPE_MODE (wide_vectype)
14089 13541 : && SCALAR_INT_MODE_P (intermediate_mode))
14090 : {
14091 : /* If the input and result modes are the same, a different optab
14092 : is needed where we pass in the number of units in vectype. */
14093 : optab3 = vec_unpacks_sbool_lo_optab;
14094 : optab4 = vec_unpacks_sbool_hi_optab;
14095 : }
14096 : else
14097 : {
14098 12919 : optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
14099 12919 : optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
14100 : }
14101 :
14102 13230 : if (!optab3 || !optab4
14103 13230 : || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
14104 13190 : || insn_data[icode1].operand[0].mode != intermediate_mode
14105 13190 : || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
14106 13190 : || insn_data[icode2].operand[0].mode != intermediate_mode
14107 13190 : || ((icode1 = optab_handler (optab3, intermediate_mode))
14108 : == CODE_FOR_nothing)
14109 26032 : || ((icode2 = optab_handler (optab4, intermediate_mode))
14110 : == CODE_FOR_nothing))
14111 : break;
14112 :
14113 12802 : interm_types->quick_push (intermediate_type);
14114 12802 : (*multi_step_cvt)++;
14115 :
14116 12802 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
14117 12802 : && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
14118 : {
14119 11553 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14120 : return true;
14121 3779 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
14122 : TYPE_VECTOR_SUBPARTS (wide_vectype) * 2))
14123 : return true;
14124 : }
14125 :
14126 1331 : prev_type = intermediate_type;
14127 1331 : prev_mode = intermediate_mode;
14128 : }
14129 :
14130 428 : interm_types->release ();
14131 428 : return false;
14132 : }
14133 :
14134 :
14135 : /* Function supportable_narrowing_operation
14136 :
14137 : Check whether an operation represented by the code CODE is a
14138 : narrowing operation that is supported by the target platform in
14139 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14140 : and producing a result of type VECTYPE_OUT).
14141 :
14142 : Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14143 : and FLOAT. This function checks if these operations are supported by
14144 : the target platform directly via vector tree-codes.
14145 :
14146 : Output:
14147 : - CODE1 is the code of a vector operation to be used when
14148 : vectorizing the operation, if available.
14149 : - MULTI_STEP_CVT determines the number of required intermediate steps in
14150 : case of multi-step conversion (like int->short->char - in that case
14151 : MULTI_STEP_CVT will be 1).
14152 : - INTERM_TYPES contains the intermediate type required to perform the
14153 : narrowing operation (short in the above example). */
14154 :
14155 : bool
14156 43707 : supportable_narrowing_operation (code_helper code,
14157 : tree vectype_out, tree vectype_in,
14158 : code_helper *code1, int *multi_step_cvt,
14159 : vec<tree> *interm_types)
14160 : {
14161 43707 : machine_mode vec_mode;
14162 43707 : enum insn_code icode1;
14163 43707 : optab optab1, interm_optab;
14164 43707 : tree vectype = vectype_in;
14165 43707 : tree narrow_vectype = vectype_out;
14166 43707 : enum tree_code c1;
14167 43707 : tree intermediate_type, prev_type;
14168 43707 : machine_mode intermediate_mode, prev_mode;
14169 43707 : int i;
14170 43707 : unsigned HOST_WIDE_INT n_elts;
14171 43707 : bool uns;
14172 :
14173 43707 : if (!code.is_tree_code ())
14174 : return false;
14175 :
14176 43707 : *multi_step_cvt = 0;
14177 43707 : switch ((tree_code) code)
14178 : {
14179 42199 : CASE_CONVERT:
14180 42199 : c1 = VEC_PACK_TRUNC_EXPR;
14181 42199 : if (VECTOR_BOOLEAN_TYPE_P (narrow_vectype)
14182 11582 : && VECTOR_BOOLEAN_TYPE_P (vectype)
14183 11582 : && SCALAR_INT_MODE_P (TYPE_MODE (vectype))
14184 5257 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&n_elts)
14185 47456 : && n_elts < BITS_PER_UNIT)
14186 : optab1 = vec_pack_sbool_trunc_optab;
14187 : else
14188 39714 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14189 : break;
14190 :
14191 570 : case FIX_TRUNC_EXPR:
14192 570 : c1 = VEC_PACK_FIX_TRUNC_EXPR;
14193 : /* The signedness is determined from output operand. */
14194 570 : optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
14195 570 : break;
14196 :
14197 938 : case FLOAT_EXPR:
14198 938 : c1 = VEC_PACK_FLOAT_EXPR;
14199 938 : optab1 = optab_for_tree_code (c1, vectype, optab_default);
14200 938 : break;
14201 :
14202 0 : default:
14203 0 : gcc_unreachable ();
14204 : }
14205 :
14206 43707 : if (!optab1)
14207 : return false;
14208 :
14209 43707 : vec_mode = TYPE_MODE (vectype);
14210 43707 : if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
14211 : return false;
14212 :
14213 38446 : *code1 = c1;
14214 :
14215 38446 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14216 : {
14217 23969 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14218 : return true;
14219 : /* For scalar masks we may have different boolean
14220 : vector types having the same QImode. Thus we
14221 : add additional check for elements number. */
14222 5822 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
14223 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14224 : return true;
14225 : }
14226 :
14227 14628 : if (code == FLOAT_EXPR)
14228 : return false;
14229 :
14230 : /* Check if it's a multi-step conversion that can be done using intermediate
14231 : types. */
14232 14628 : prev_mode = vec_mode;
14233 14628 : prev_type = vectype;
14234 14628 : if (code == FIX_TRUNC_EXPR)
14235 94 : uns = TYPE_UNSIGNED (vectype_out);
14236 : else
14237 14534 : uns = TYPE_UNSIGNED (vectype);
14238 :
14239 : /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14240 : conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14241 : costly than signed. */
14242 14628 : if (code == FIX_TRUNC_EXPR && uns)
14243 : {
14244 28 : enum insn_code icode2;
14245 :
14246 28 : intermediate_type
14247 28 : = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
14248 28 : interm_optab
14249 28 : = optab_for_tree_code (c1, intermediate_type, optab_default);
14250 28 : if (interm_optab != unknown_optab
14251 28 : && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
14252 28 : && insn_data[icode1].operand[0].mode
14253 28 : == insn_data[icode2].operand[0].mode)
14254 : {
14255 : uns = false;
14256 : optab1 = interm_optab;
14257 : icode1 = icode2;
14258 : }
14259 : }
14260 :
14261 : /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14262 : intermediate steps in promotion sequence. We try
14263 : MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14264 14628 : interm_types->create (MAX_INTERM_CVT_STEPS);
14265 31338 : for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
14266 : {
14267 16710 : intermediate_mode = insn_data[icode1].operand[0].mode;
14268 16710 : if (VECTOR_BOOLEAN_TYPE_P (prev_type))
14269 6997 : intermediate_type
14270 6997 : = vect_double_mask_nunits (prev_type, intermediate_mode);
14271 : else
14272 9713 : intermediate_type
14273 9713 : = lang_hooks.types.type_for_mode (intermediate_mode, uns);
14274 16710 : if (VECTOR_BOOLEAN_TYPE_P (intermediate_type)
14275 6997 : && VECTOR_BOOLEAN_TYPE_P (prev_type)
14276 6997 : && SCALAR_INT_MODE_P (prev_mode)
14277 3128 : && TYPE_VECTOR_SUBPARTS (intermediate_type).is_constant (&n_elts)
14278 19838 : && n_elts < BITS_PER_UNIT)
14279 : interm_optab = vec_pack_sbool_trunc_optab;
14280 : else
14281 16358 : interm_optab
14282 16358 : = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
14283 : optab_default);
14284 352 : if (!interm_optab
14285 16710 : || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
14286 16710 : || insn_data[icode1].operand[0].mode != intermediate_mode
14287 33068 : || ((icode1 = optab_handler (interm_optab, intermediate_mode))
14288 : == CODE_FOR_nothing))
14289 : break;
14290 :
14291 15795 : interm_types->quick_push (intermediate_type);
14292 15795 : (*multi_step_cvt)++;
14293 :
14294 15795 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
14295 : {
14296 13713 : if (!VECTOR_BOOLEAN_TYPE_P (vectype))
14297 : return true;
14298 4925 : if (known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
14299 : TYPE_VECTOR_SUBPARTS (narrow_vectype)))
14300 : return true;
14301 : }
14302 :
14303 2082 : prev_mode = intermediate_mode;
14304 2082 : prev_type = intermediate_type;
14305 2082 : optab1 = interm_optab;
14306 : }
14307 :
14308 915 : interm_types->release ();
14309 915 : return false;
14310 : }
14311 :
14312 : /* Function supportable_indirect_convert_operation
14313 :
14314 : Check whether an operation represented by the code CODE is single or multi
14315 : operations that are supported by the target platform in
14316 : vector form (i.e., when operating on arguments of type VECTYPE_IN
14317 : producing a result of type VECTYPE_OUT).
14318 :
14319 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14320 : This function checks if these operations are supported
14321 : by the target platform directly (via vector tree-codes).
14322 :
14323 : Output:
14324 : - converts contains some pairs to perform the convert operation,
14325 : the pair's first is the intermediate type, and its second is the code of
14326 : a vector operation to be used when converting the operation from the
14327 : previous type to the intermediate type. */
14328 : bool
14329 93406 : supportable_indirect_convert_operation (code_helper code,
14330 : tree vectype_out,
14331 : tree vectype_in,
14332 : vec<std::pair<tree, tree_code> > &converts,
14333 : slp_tree slp_op0)
14334 : {
14335 93406 : bool found_mode = false;
14336 93406 : scalar_mode lhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_out));
14337 93406 : scalar_mode rhs_mode = GET_MODE_INNER (TYPE_MODE (vectype_in));
14338 93406 : tree_code code1, code2;
14339 :
14340 93406 : tree cvt_type = NULL_TREE;
14341 93406 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (vectype_in);
14342 :
14343 93406 : if (supportable_convert_operation ((tree_code) code,
14344 : vectype_out,
14345 : vectype_in))
14346 : {
14347 85859 : converts.safe_push (std::make_pair (vectype_out, (tree_code) code));
14348 85859 : return true;
14349 : }
14350 :
14351 : /* For conversions between mask types where the destination has
14352 : a data mode attempt a vcond_mask conversion. */
14353 7547 : if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
14354 143 : && VECTOR_BOOLEAN_TYPE_P (vectype_out)
14355 7690 : && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
14356 : {
14357 142 : tree scalar_datatype
14358 142 : = build_nonstandard_integer_type (element_precision (vectype_out), 0);
14359 142 : tree datatype_out = build_vector_type_for_mode (scalar_datatype,
14360 142 : TYPE_MODE (vectype_out));
14361 142 : if (expand_vec_cond_expr_p (datatype_out, vectype_in))
14362 : {
14363 2 : converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
14364 2 : return true;
14365 : }
14366 : }
14367 :
14368 : /* For conversions between float and integer types try whether
14369 : we can use intermediate signed integer types to support the
14370 : conversion. */
14371 15090 : if (GET_MODE_SIZE (lhs_mode) != GET_MODE_SIZE (rhs_mode)
14372 7545 : && (code == FLOAT_EXPR
14373 4906 : || (code == FIX_TRUNC_EXPR && !flag_trapping_math)))
14374 : {
14375 1592 : bool demotion = GET_MODE_SIZE (rhs_mode) > GET_MODE_SIZE (lhs_mode);
14376 796 : bool float_expr_p = code == FLOAT_EXPR;
14377 796 : unsigned short target_size;
14378 796 : scalar_mode intermediate_mode;
14379 796 : if (demotion)
14380 : {
14381 99 : intermediate_mode = lhs_mode;
14382 99 : target_size = GET_MODE_SIZE (rhs_mode);
14383 : }
14384 : else
14385 : {
14386 697 : target_size = GET_MODE_SIZE (lhs_mode);
14387 697 : if (!int_mode_for_size
14388 697 : (GET_MODE_BITSIZE (rhs_mode), 0).exists (&intermediate_mode))
14389 142 : return false;
14390 : }
14391 796 : code1 = float_expr_p ? (tree_code) code : NOP_EXPR;
14392 729 : code2 = float_expr_p ? NOP_EXPR : (tree_code) code;
14393 796 : opt_scalar_mode mode_iter;
14394 2483 : FOR_EACH_2XWIDER_MODE (mode_iter, intermediate_mode)
14395 : {
14396 2483 : intermediate_mode = mode_iter.require ();
14397 :
14398 4966 : if (GET_MODE_SIZE (intermediate_mode) > target_size)
14399 : break;
14400 :
14401 1882 : scalar_mode cvt_mode;
14402 1882 : if (!int_mode_for_size
14403 1882 : (GET_MODE_BITSIZE (intermediate_mode), 0).exists (&cvt_mode))
14404 : break;
14405 :
14406 1852 : cvt_type = build_nonstandard_integer_type
14407 1852 : (GET_MODE_BITSIZE (cvt_mode), 0);
14408 :
14409 : /* Check if the intermediate type can hold OP0's range.
14410 : When converting from float to integer this is not necessary
14411 : because values that do not fit the (smaller) target type are
14412 : unspecified anyway. */
14413 1852 : if (demotion && float_expr_p)
14414 : {
14415 23 : wide_int op_min_value, op_max_value;
14416 : /* For vector form, it looks like slp_op0 doesn't have RANGE_INFO.
14417 : In the future, if it is supported, changes may need to be made
14418 : to this part, such as checking the RANGE of each element
14419 : in the vector. */
14420 23 : if (slp_op0)
14421 : {
14422 13 : tree def;
14423 : /* ??? Merge ranges in case of more than one lane. */
14424 13 : if (SLP_TREE_LANES (slp_op0) != 1
14425 0 : || !(def = vect_get_slp_scalar_def (slp_op0, 0))
14426 13 : || !vect_get_range_info (def,
14427 : &op_min_value, &op_max_value))
14428 : break;
14429 : }
14430 : else
14431 : break;
14432 :
14433 0 : if (cvt_type == NULL_TREE
14434 0 : || (wi::min_precision (op_max_value, SIGNED)
14435 0 : > TYPE_PRECISION (cvt_type))
14436 0 : || (wi::min_precision (op_min_value, SIGNED)
14437 0 : > TYPE_PRECISION (cvt_type)))
14438 0 : continue;
14439 23 : }
14440 :
14441 1829 : cvt_type = get_related_vectype_for_scalar_type (TYPE_MODE (vectype_in),
14442 : cvt_type,
14443 : nelts);
14444 : /* This should only happened for SLP as long as loop vectorizer
14445 : only supports same-sized vector. */
14446 3516 : if (cvt_type == NULL_TREE
14447 3658 : || maybe_ne (TYPE_VECTOR_SUBPARTS (cvt_type), nelts)
14448 1829 : || !supportable_convert_operation ((tree_code) code1,
14449 : vectype_out, cvt_type)
14450 2564 : || !supportable_convert_operation ((tree_code) code2,
14451 : cvt_type, vectype_in))
14452 1687 : continue;
14453 :
14454 : found_mode = true;
14455 : break;
14456 : }
14457 :
14458 796 : if (found_mode)
14459 : {
14460 142 : converts.safe_push (std::make_pair (cvt_type, code2));
14461 142 : if (TYPE_MODE (cvt_type) != TYPE_MODE (vectype_out))
14462 142 : converts.safe_push (std::make_pair (vectype_out, code1));
14463 : return true;
14464 : }
14465 : }
14466 : return false;
14467 : }
14468 :
14469 : /* Generate and return a vector mask of MASK_TYPE such that
14470 : mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14471 : Add the statements to SEQ. */
14472 :
14473 : tree
14474 0 : vect_gen_while (gimple_seq *seq, tree mask_type, tree start_index,
14475 : tree end_index, const char *name)
14476 : {
14477 0 : tree cmp_type = TREE_TYPE (start_index);
14478 0 : gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
14479 : cmp_type, mask_type,
14480 : OPTIMIZE_FOR_SPEED));
14481 0 : gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
14482 : start_index, end_index,
14483 : build_zero_cst (mask_type));
14484 0 : tree tmp;
14485 0 : if (name)
14486 0 : tmp = make_temp_ssa_name (mask_type, NULL, name);
14487 : else
14488 0 : tmp = make_ssa_name (mask_type);
14489 0 : gimple_call_set_lhs (call, tmp);
14490 0 : gimple_seq_add_stmt (seq, call);
14491 0 : return tmp;
14492 : }
14493 :
14494 : /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14495 : J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14496 :
14497 : tree
14498 0 : vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
14499 : tree end_index)
14500 : {
14501 0 : tree tmp = vect_gen_while (seq, mask_type, start_index, end_index);
14502 0 : return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
14503 : }
14504 :
14505 : /* Try to compute the vector types required to vectorize STMT_INFO,
14506 : returning true on success and false if vectorization isn't possible.
14507 : If GROUP_SIZE is nonzero and we're performing BB vectorization,
14508 : take sure that the number of elements in the vectors is no bigger
14509 : than GROUP_SIZE.
14510 :
14511 : On success:
14512 :
14513 : - Set *STMT_VECTYPE_OUT to:
14514 : - NULL_TREE if the statement doesn't need to be vectorized;
14515 : - the equivalent of STMT_VINFO_VECTYPE otherwise. */
14516 :
14517 : opt_result
14518 6100601 : vect_get_vector_types_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
14519 : tree *stmt_vectype_out,
14520 : unsigned int group_size)
14521 : {
14522 6100601 : gimple *stmt = stmt_info->stmt;
14523 :
14524 : /* For BB vectorization, we should always have a group size once we've
14525 : constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14526 : are tentative requests during things like early data reference
14527 : analysis and pattern recognition. */
14528 6100601 : if (is_a <bb_vec_info> (vinfo))
14529 4877022 : gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0);
14530 : else
14531 : group_size = 0;
14532 :
14533 6100601 : *stmt_vectype_out = NULL_TREE;
14534 :
14535 6100601 : if (gimple_get_lhs (stmt) == NULL_TREE
14536 : /* Allow vector conditionals through here. */
14537 2610 : && !is_a <gcond *> (stmt)
14538 : /* MASK_STORE and friends have no lhs, but are ok. */
14539 6105801 : && !(is_gimple_call (stmt)
14540 2610 : && gimple_call_internal_p (stmt)
14541 2590 : && internal_store_fn_p (gimple_call_internal_fn (stmt))))
14542 : {
14543 20 : if (is_a <gcall *> (stmt))
14544 : {
14545 : /* Ignore calls with no lhs. These must be calls to
14546 : #pragma omp simd functions, and what vectorization factor
14547 : it really needs can't be determined until
14548 : vectorizable_simd_clone_call. */
14549 20 : if (dump_enabled_p ())
14550 18 : dump_printf_loc (MSG_NOTE, vect_location,
14551 : "defer to SIMD clone analysis.\n");
14552 20 : return opt_result::success ();
14553 : }
14554 :
14555 0 : return opt_result::failure_at (stmt,
14556 : "not vectorized: irregular stmt: %G", stmt);
14557 : }
14558 :
14559 6100581 : tree vectype;
14560 6100581 : tree scalar_type = NULL_TREE;
14561 6100581 : if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
14562 : {
14563 1651317 : vectype = STMT_VINFO_VECTYPE (stmt_info);
14564 1651317 : if (dump_enabled_p ())
14565 81498 : dump_printf_loc (MSG_NOTE, vect_location,
14566 : "precomputed vectype: %T\n", vectype);
14567 : }
14568 4449264 : else if (vect_use_mask_type_p (stmt_info))
14569 : {
14570 228856 : unsigned int precision = stmt_info->mask_precision;
14571 228856 : scalar_type = build_nonstandard_integer_type (precision, 1);
14572 228856 : vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
14573 228856 : if (!vectype)
14574 0 : return opt_result::failure_at (stmt, "not vectorized: unsupported"
14575 : " data-type %T\n", scalar_type);
14576 228856 : if (dump_enabled_p ())
14577 4875 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14578 : }
14579 : else
14580 : {
14581 : /* If we got here with a gcond it means that the target had no available vector
14582 : mode for the scalar type. We can't vectorize so abort. */
14583 4220408 : if (is_a <gcond *> (stmt))
14584 0 : return opt_result::failure_at (stmt,
14585 : "not vectorized:"
14586 : " unsupported data-type for gcond %T\n",
14587 : scalar_type);
14588 :
14589 4220408 : if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
14590 1510492 : scalar_type = TREE_TYPE (DR_REF (dr));
14591 : else
14592 2709916 : scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
14593 :
14594 4220408 : if (dump_enabled_p ())
14595 : {
14596 65843 : if (group_size)
14597 9129 : dump_printf_loc (MSG_NOTE, vect_location,
14598 : "get vectype for scalar type (group size %d):"
14599 : " %T\n", group_size, scalar_type);
14600 : else
14601 56714 : dump_printf_loc (MSG_NOTE, vect_location,
14602 : "get vectype for scalar type: %T\n", scalar_type);
14603 : }
14604 4220408 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
14605 4220408 : if (!vectype)
14606 229172 : return opt_result::failure_at (stmt,
14607 : "not vectorized:"
14608 : " unsupported data-type %T\n",
14609 : scalar_type);
14610 :
14611 3991236 : if (dump_enabled_p ())
14612 65585 : dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
14613 : }
14614 :
14615 4301590 : if (scalar_type && VECTOR_MODE_P (TYPE_MODE (scalar_type)))
14616 0 : return opt_result::failure_at (stmt,
14617 : "not vectorized: vector stmt in loop:%G",
14618 : stmt);
14619 :
14620 5871409 : *stmt_vectype_out = vectype;
14621 5871409 : return opt_result::success ();
14622 : }
14623 :
14624 : /* Generate and return statement sequence that sets vector length LEN that is:
14625 :
14626 : min_of_start_and_end = min (START_INDEX, END_INDEX);
14627 : left_len = END_INDEX - min_of_start_and_end;
14628 : rhs = min (left_len, LEN_LIMIT);
14629 : LEN = rhs;
14630 :
14631 : Note: the cost of the code generated by this function is modeled
14632 : by vect_estimate_min_profitable_iters, so changes here may need
14633 : corresponding changes there. */
14634 :
14635 : gimple_seq
14636 0 : vect_gen_len (tree len, tree start_index, tree end_index, tree len_limit)
14637 : {
14638 0 : gimple_seq stmts = NULL;
14639 0 : tree len_type = TREE_TYPE (len);
14640 0 : gcc_assert (TREE_TYPE (start_index) == len_type);
14641 :
14642 0 : tree min = gimple_build (&stmts, MIN_EXPR, len_type, start_index, end_index);
14643 0 : tree left_len = gimple_build (&stmts, MINUS_EXPR, len_type, end_index, min);
14644 0 : tree rhs = gimple_build (&stmts, MIN_EXPR, len_type, left_len, len_limit);
14645 0 : gimple* stmt = gimple_build_assign (len, rhs);
14646 0 : gimple_seq_add_stmt (&stmts, stmt);
14647 :
14648 0 : return stmts;
14649 : }
14650 :
|
