Line data Source code
1 : /* SLP - Basic Block Vectorization
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #define INCLUDE_ALGORITHM
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "optabs-tree.h"
34 : #include "insn-config.h"
35 : #include "recog.h" /* FIXME: for insn_data */
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "gimple-iterator.h"
39 : #include "cfgloop.h"
40 : #include "tree-vectorizer.h"
41 : #include "langhooks.h"
42 : #include "gimple-walk.h"
43 : #include "dbgcnt.h"
44 : #include "tree-vector-builder.h"
45 : #include "vec-perm-indices.h"
46 : #include "gimple-fold.h"
47 : #include "internal-fn.h"
48 : #include "dump-context.h"
49 : #include "cfganal.h"
50 : #include "tree-eh.h"
51 : #include "tree-cfg.h"
52 : #include "alloc-pool.h"
53 : #include "sreal.h"
54 : #include "predict.h"
55 :
56 : #define REDUC_GROUP_FIRST_ELEMENT(S) \
57 : (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
58 :
59 : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
60 : load_permutation_t &,
61 : const vec<tree> &,
62 : gimple_stmt_iterator *,
63 : poly_uint64, bool, bool,
64 : unsigned *,
65 : unsigned * = nullptr,
66 : bool = false);
67 : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
68 : slp_tree, lane_permutation_t &,
69 : vec<slp_tree> &, bool);
70 : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
71 : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
72 :
73 : static object_allocator<_slp_tree> *slp_tree_pool;
74 : static slp_tree slp_first_node;
75 :
76 : void
77 1124071 : vect_slp_init (void)
78 : {
79 1124071 : slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
80 1124071 : }
81 :
82 : void
83 1124071 : vect_slp_fini (void)
84 : {
85 1813456 : while (slp_first_node)
86 689385 : delete slp_first_node;
87 2248142 : delete slp_tree_pool;
88 1124071 : slp_tree_pool = NULL;
89 1124071 : }
90 :
91 : void *
92 8295382 : _slp_tree::operator new (size_t n)
93 : {
94 8295382 : gcc_assert (n == sizeof (_slp_tree));
95 8295382 : return slp_tree_pool->allocate_raw ();
96 : }
97 :
98 : void
99 8295382 : _slp_tree::operator delete (void *node, size_t n)
100 : {
101 8295382 : gcc_assert (n == sizeof (_slp_tree));
102 8295382 : slp_tree_pool->remove_raw (node);
103 8295382 : }
104 :
105 :
106 : /* Initialize a SLP node. */
107 :
108 8295382 : _slp_tree::_slp_tree ()
109 : {
110 8295382 : this->prev_node = NULL;
111 8295382 : if (slp_first_node)
112 7270571 : slp_first_node->prev_node = this;
113 8295382 : this->next_node = slp_first_node;
114 8295382 : slp_first_node = this;
115 8295382 : SLP_TREE_SCALAR_STMTS (this) = vNULL;
116 8295382 : SLP_TREE_SCALAR_OPS (this) = vNULL;
117 8295382 : SLP_TREE_LIVE_LANES (this) = vNULL;
118 8295382 : SLP_TREE_VEC_DEFS (this) = vNULL;
119 8295382 : SLP_TREE_CHILDREN (this) = vNULL;
120 8295382 : SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
121 8295382 : SLP_TREE_LANE_PERMUTATION (this) = vNULL;
122 8295382 : SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
123 8295382 : SLP_TREE_CODE (this) = ERROR_MARK;
124 8295382 : SLP_TREE_GS_SCALE (this) = 0;
125 8295382 : SLP_TREE_GS_BASE (this) = NULL_TREE;
126 8295382 : this->ldst_lanes = false;
127 8295382 : this->avoid_stlf_fail = false;
128 8295382 : SLP_TREE_VECTYPE (this) = NULL_TREE;
129 8295382 : SLP_TREE_REPRESENTATIVE (this) = NULL;
130 8295382 : this->cycle_info.id = -1;
131 8295382 : this->cycle_info.reduc_idx = -1;
132 8295382 : SLP_TREE_REF_COUNT (this) = 1;
133 8295382 : this->failed = NULL;
134 8295382 : this->max_nunits = 1;
135 8295382 : this->lanes = 0;
136 8295382 : SLP_TREE_TYPE (this) = undef_vec_info_type;
137 8295382 : this->data = NULL;
138 8295382 : }
139 :
140 : /* Tear down a SLP node. */
141 :
142 8295382 : _slp_tree::~_slp_tree ()
143 : {
144 8295382 : if (this->prev_node)
145 5094181 : this->prev_node->next_node = this->next_node;
146 : else
147 3201201 : slp_first_node = this->next_node;
148 8295382 : if (this->next_node)
149 6279980 : this->next_node->prev_node = this->prev_node;
150 8295382 : SLP_TREE_CHILDREN (this).release ();
151 8295382 : SLP_TREE_SCALAR_STMTS (this).release ();
152 8295382 : SLP_TREE_SCALAR_OPS (this).release ();
153 8295382 : SLP_TREE_LIVE_LANES (this).release ();
154 8295382 : SLP_TREE_VEC_DEFS (this).release ();
155 8295382 : SLP_TREE_LOAD_PERMUTATION (this).release ();
156 8295382 : SLP_TREE_LANE_PERMUTATION (this).release ();
157 8295382 : if (this->failed)
158 2251103 : free (failed);
159 8295382 : if (this->data)
160 1273928 : delete this->data;
161 8295382 : }
162 :
163 : /* Push the single SSA definition in DEF to the vector of vector defs. */
164 :
165 : void
166 532756 : _slp_tree::push_vec_def (gimple *def)
167 : {
168 532756 : if (gphi *phi = dyn_cast <gphi *> (def))
169 59487 : vec_defs.quick_push (gimple_phi_result (phi));
170 : else
171 : {
172 473269 : def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
173 473269 : vec_defs.quick_push (get_def_from_ptr (defop));
174 : }
175 532756 : }
176 :
177 : /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
178 :
179 : void
180 15603971 : vect_free_slp_tree (slp_tree node)
181 : {
182 15603971 : int i;
183 15603971 : slp_tree child;
184 :
185 15603971 : if (--SLP_TREE_REF_COUNT (node) != 0)
186 15603971 : return;
187 :
188 11774058 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
189 4168061 : if (child)
190 3788717 : vect_free_slp_tree (child);
191 :
192 7605997 : delete node;
193 : }
194 :
195 : /* Return a location suitable for dumpings related to the SLP instance. */
196 :
197 : dump_user_location_t
198 3533849 : _slp_instance::location () const
199 : {
200 3533849 : if (!root_stmts.is_empty ())
201 412030 : return root_stmts[0]->stmt;
202 : else
203 3121819 : return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
204 : }
205 :
206 :
207 : /* Free the memory allocated for the SLP instance. */
208 :
209 : void
210 1620171 : vect_free_slp_instance (slp_instance instance)
211 : {
212 1620171 : vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
213 1620171 : SLP_INSTANCE_LOADS (instance).release ();
214 1620171 : SLP_INSTANCE_ROOT_STMTS (instance).release ();
215 1620171 : SLP_INSTANCE_REMAIN_DEFS (instance).release ();
216 1620171 : instance->subgraph_entries.release ();
217 1620171 : instance->cost_vec.release ();
218 1620171 : free (instance);
219 1620171 : }
220 :
221 :
222 : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
223 : for a permute node or else ERROR_MARK. */
224 :
225 : slp_tree
226 97525 : vect_create_new_slp_node (unsigned nops, tree_code code)
227 : {
228 97525 : gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
229 97525 : slp_tree node = new _slp_tree;
230 97525 : SLP_TREE_SCALAR_STMTS (node) = vNULL;
231 97525 : SLP_TREE_CHILDREN (node).create (nops);
232 97525 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
233 97525 : SLP_TREE_CODE (node) = code;
234 97525 : return node;
235 : }
236 :
237 : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children. */
238 :
239 : static slp_tree
240 3969114 : vect_create_new_slp_node (slp_tree node,
241 : vec<stmt_vec_info> scalar_stmts, unsigned nops)
242 : {
243 3969114 : SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
244 3969114 : SLP_TREE_CHILDREN (node).create (nops);
245 3969114 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
246 3969114 : SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
247 3969114 : SLP_TREE_LANES (node) = scalar_stmts.length ();
248 3969114 : return node;
249 : }
250 :
251 : /* Create an SLP node for SCALAR_STMTS and NOPS children. */
252 :
253 : static slp_tree
254 8377 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
255 : {
256 8377 : return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
257 : }
258 :
259 : /* Create a vect_external_def SLP node inplace at NODE for scalar
260 : operands OPS. */
261 :
262 : static slp_tree
263 1965688 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
264 : {
265 1965688 : SLP_TREE_SCALAR_OPS (node) = ops;
266 1965688 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
267 0 : SLP_TREE_LANES (node) = ops.length ();
268 1965688 : return node;
269 : }
270 :
271 : /* Create a vect_external_def SLP node for scalar operands OPS. */
272 :
273 : static slp_tree
274 1965688 : vect_create_new_slp_node (vec<tree> ops)
275 : {
276 1965688 : return vect_create_new_slp_node (new _slp_tree, ops);
277 : }
278 :
279 :
280 : /* This structure is used in creation of an SLP tree. Each instance
281 : corresponds to the same operand in a group of scalar stmts in an SLP
282 : node. */
283 : typedef struct _slp_oprnd_info
284 : {
285 : /* Def-stmts for the operands. */
286 : vec<stmt_vec_info> def_stmts;
287 : /* Operands. */
288 : vec<tree> ops;
289 : /* Information about the first statement, its vector def-type, type, the
290 : operand itself in case it's constant, and an indication if it's a pattern
291 : stmt and gather/scatter info. */
292 : tree first_op_type;
293 : enum vect_def_type first_dt;
294 : bool any_pattern;
295 : bool first_gs_p;
296 : gather_scatter_info first_gs_info;
297 : } *slp_oprnd_info;
298 :
299 :
300 : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
301 : operand. */
302 : static vec<slp_oprnd_info>
303 3531379 : vect_create_oprnd_info (int nops, int group_size)
304 : {
305 3531379 : int i;
306 3531379 : slp_oprnd_info oprnd_info;
307 3531379 : vec<slp_oprnd_info> oprnds_info;
308 :
309 3531379 : oprnds_info.create (nops);
310 12708069 : for (i = 0; i < nops; i++)
311 : {
312 5645311 : oprnd_info = XNEW (struct _slp_oprnd_info);
313 5645311 : oprnd_info->def_stmts.create (group_size);
314 5645311 : oprnd_info->ops.create (group_size);
315 5645311 : oprnd_info->first_dt = vect_uninitialized_def;
316 5645311 : oprnd_info->first_op_type = NULL_TREE;
317 5645311 : oprnd_info->any_pattern = false;
318 5645311 : oprnd_info->first_gs_p = false;
319 5645311 : oprnds_info.quick_push (oprnd_info);
320 : }
321 :
322 3531379 : return oprnds_info;
323 : }
324 :
325 :
326 : /* Free operands info. */
327 :
328 : static void
329 3531379 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
330 : {
331 3531379 : int i;
332 3531379 : slp_oprnd_info oprnd_info;
333 :
334 9176690 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
335 : {
336 5645311 : oprnd_info->def_stmts.release ();
337 5645311 : oprnd_info->ops.release ();
338 5645311 : XDELETE (oprnd_info);
339 : }
340 :
341 3531379 : oprnds_info.release ();
342 3531379 : }
343 :
344 : /* Return the execution frequency of NODE (so that a higher value indicates
345 : a "more important" node when optimizing for speed). */
346 :
347 : static sreal
348 3655518 : vect_slp_node_weight (slp_tree node)
349 : {
350 3655518 : stmt_vec_info stmt_info = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
351 3655518 : basic_block bb = gimple_bb (stmt_info->stmt);
352 3655518 : return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
353 : }
354 :
355 : /* Return true if STMTS contains a pattern statement. */
356 :
357 : static bool
358 31046 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
359 : {
360 31046 : stmt_vec_info stmt_info;
361 31046 : unsigned int i;
362 89446 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
363 66393 : if (stmt_info && is_pattern_stmt_p (stmt_info))
364 : return true;
365 : return false;
366 : }
367 :
368 : /* Return true when all lanes in the external or constant NODE have
369 : the same value. */
370 :
371 : static bool
372 648698 : vect_slp_tree_uniform_p (slp_tree node)
373 : {
374 648698 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
375 : || SLP_TREE_DEF_TYPE (node) == vect_external_def);
376 :
377 : /* Pre-existing vectors. */
378 1137894 : if (SLP_TREE_SCALAR_OPS (node).is_empty ())
379 : return false;
380 :
381 : unsigned i;
382 : tree op, first = NULL_TREE;
383 1490190 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
384 1330688 : if (!first)
385 : first = op;
386 681990 : else if (!operand_equal_p (first, op, 0))
387 : return false;
388 :
389 : return true;
390 : }
391 :
392 : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
393 : that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
394 : of the chain. */
395 :
396 : int
397 749780 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
398 : stmt_vec_info first_stmt_info)
399 : {
400 749780 : stmt_vec_info next_stmt_info = first_stmt_info;
401 749780 : int result = 0;
402 :
403 749780 : if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
404 : return -1;
405 :
406 2129711 : do
407 : {
408 2129711 : if (next_stmt_info == stmt_info)
409 : return result;
410 1379931 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
411 1379931 : if (next_stmt_info)
412 1379931 : result += DR_GROUP_GAP (next_stmt_info);
413 : }
414 1379931 : while (next_stmt_info);
415 :
416 : return -1;
417 : }
418 :
419 : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
420 : using the method implemented by duplicate_and_interleave. Return true
421 : if so, returning the number of intermediate vectors in *NVECTORS_OUT
422 : (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
423 : (if nonnull). */
424 :
425 : bool
426 0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
427 : tree elt_type, unsigned int *nvectors_out,
428 : tree *vector_type_out,
429 : tree *permutes)
430 : {
431 0 : tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
432 0 : if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
433 0 : return false;
434 :
435 0 : machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
436 0 : poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
437 0 : unsigned int nvectors = 1;
438 0 : for (;;)
439 : {
440 0 : scalar_int_mode int_mode;
441 0 : poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
442 0 : if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
443 : {
444 : /* Get the natural vector type for this SLP group size. */
445 0 : tree int_type = build_nonstandard_integer_type
446 0 : (GET_MODE_BITSIZE (int_mode), 1);
447 0 : tree vector_type
448 0 : = get_vectype_for_scalar_type (vinfo, int_type, count);
449 0 : poly_int64 half_nelts;
450 0 : if (vector_type
451 0 : && VECTOR_MODE_P (TYPE_MODE (vector_type))
452 0 : && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
453 : GET_MODE_SIZE (base_vector_mode))
454 0 : && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
455 : 2, &half_nelts))
456 : {
457 : /* Try fusing consecutive sequences of COUNT / NVECTORS elements
458 : together into elements of type INT_TYPE and using the result
459 : to build NVECTORS vectors. */
460 0 : poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
461 0 : vec_perm_builder sel1 (nelts, 2, 3);
462 0 : vec_perm_builder sel2 (nelts, 2, 3);
463 :
464 0 : for (unsigned int i = 0; i < 3; ++i)
465 : {
466 0 : sel1.quick_push (i);
467 0 : sel1.quick_push (i + nelts);
468 0 : sel2.quick_push (half_nelts + i);
469 0 : sel2.quick_push (half_nelts + i + nelts);
470 : }
471 0 : vec_perm_indices indices1 (sel1, 2, nelts);
472 0 : vec_perm_indices indices2 (sel2, 2, nelts);
473 0 : machine_mode vmode = TYPE_MODE (vector_type);
474 0 : if (can_vec_perm_const_p (vmode, vmode, indices1)
475 0 : && can_vec_perm_const_p (vmode, vmode, indices2))
476 : {
477 0 : if (nvectors_out)
478 0 : *nvectors_out = nvectors;
479 0 : if (vector_type_out)
480 0 : *vector_type_out = vector_type;
481 0 : if (permutes)
482 : {
483 0 : permutes[0] = vect_gen_perm_mask_checked (vector_type,
484 : indices1);
485 0 : permutes[1] = vect_gen_perm_mask_checked (vector_type,
486 : indices2);
487 : }
488 0 : return true;
489 : }
490 0 : }
491 : }
492 0 : if (!multiple_p (elt_bytes, 2, &elt_bytes))
493 : return false;
494 0 : nvectors *= 2;
495 : /* We need to be able to fuse COUNT / NVECTORS elements together. */
496 0 : if (!multiple_p (count, nvectors))
497 : return false;
498 : }
499 : }
500 :
501 : /* Return true if DTA and DTB match. */
502 :
503 : static bool
504 17234384 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
505 : {
506 17234384 : return (dta == dtb
507 364628 : || ((dta == vect_external_def || dta == vect_constant_def)
508 225539 : && (dtb == vect_external_def || dtb == vect_constant_def)));
509 : }
510 :
511 : #define GATHER_SCATTER_OFFSET (-3)
512 :
513 : /* For most SLP statements, there is a one-to-one mapping between
514 : gimple arguments and child nodes. If that is not true for STMT,
515 : return an array that contains:
516 :
517 : - the number of child nodes, followed by
518 : - for each child node, the index of the argument associated with that node.
519 : The special index -1 is the first operand of an embedded comparison and
520 : the special index -2 is the second operand of an embedded comparison.
521 : The special index -3 is the offset of a gather as analyzed by
522 : vect_check_gather_scatter.
523 :
524 : SWAP is as for vect_get_and_check_slp_defs. */
525 :
526 : static const int *
527 25271897 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
528 : unsigned char swap)
529 : {
530 25271897 : static const int no_arg_map[] = { 0 };
531 25271897 : static const int arg0_map[] = { 1, 0 };
532 25271897 : static const int arg2_map[] = { 1, 2 };
533 25271897 : static const int arg2_arg3_map[] = { 2, 2, 3 };
534 25271897 : static const int arg2_arg4_map[] = { 2, 2, 4 };
535 25271897 : static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
536 25271897 : static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
537 25271897 : static const int arg3_arg2_map[] = { 2, 3, 2 };
538 25271897 : static const int op00_map[] = { 1, -1 };
539 25271897 : static const int op1_op0_map[] = { 2, 1, 0 };
540 25271897 : static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
541 25271897 : static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
542 25271897 : static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
543 25271897 : static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
544 25271897 : static const int mask_call_maps[6][7] = {
545 : { 1, 1, },
546 : { 2, 1, 2, },
547 : { 3, 1, 2, 3, },
548 : { 4, 1, 2, 3, 4, },
549 : { 5, 1, 2, 3, 4, 5, },
550 : { 6, 1, 2, 3, 4, 5, 6 },
551 : };
552 :
553 25271897 : gcc_checking_assert (!swap
554 : || !is_gimple_assign (stmt)
555 : || TREE_CODE_CLASS
556 : (gimple_assign_rhs_code (stmt)) == tcc_comparison
557 : || commutative_tree_code
558 : (gimple_assign_rhs_code (stmt)));
559 :
560 25271897 : if (auto assign = dyn_cast<const gassign *> (stmt))
561 : {
562 23720954 : tree_code code = gimple_assign_rhs_code (assign);
563 23720954 : if (code == COND_EXPR
564 23720954 : && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
565 0 : gcc_unreachable ();
566 23720954 : else if ((TREE_CODE_CLASS (code) == tcc_comparison
567 22215769 : || commutative_tree_code (code))
568 32862339 : && swap)
569 : return op1_op0_map;
570 23677883 : else if (code == VIEW_CONVERT_EXPR)
571 : return op00_map;
572 23668714 : else if (gather_scatter_p)
573 45960 : return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
574 45960 : ? off_op0_map : off_map);
575 : }
576 1550943 : else if (auto call = dyn_cast<const gcall *> (stmt))
577 : {
578 176621 : if (gimple_call_internal_p (call))
579 93637 : switch (gimple_call_internal_fn (call))
580 : {
581 16137 : case IFN_MASK_LOAD:
582 27548 : return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
583 :
584 : case IFN_GATHER_LOAD:
585 : return arg2_map;
586 :
587 0 : case IFN_MASK_GATHER_LOAD:
588 0 : case IFN_MASK_LEN_GATHER_LOAD:
589 0 : return arg2_arg5_arg6_map;
590 :
591 0 : case IFN_SCATTER_STORE:
592 0 : return arg2_arg4_map;
593 :
594 0 : case IFN_MASK_SCATTER_STORE:
595 0 : case IFN_MASK_LEN_SCATTER_STORE:
596 0 : return arg2_arg4_arg5_map;
597 :
598 9125 : case IFN_MASK_STORE:
599 16828 : return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
600 :
601 996 : case IFN_MASK_CALL:
602 996 : {
603 996 : unsigned nargs = gimple_call_num_args (call);
604 996 : if (nargs >= 2 && nargs <= 7)
605 996 : return mask_call_maps[nargs-2];
606 : else
607 : return nullptr;
608 : }
609 :
610 278 : case IFN_CLZ:
611 278 : case IFN_CTZ:
612 278 : return arg0_map;
613 :
614 7290 : case IFN_GOMP_SIMD_LANE:
615 7290 : return no_arg_map;
616 :
617 : default:
618 : break;
619 : }
620 : }
621 : return nullptr;
622 : }
623 :
624 : static const int *
625 25250562 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
626 : {
627 0 : return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
628 0 : swap);
629 : }
630 :
631 : /* Return the SLP node child index for operand OP of STMT. */
632 :
633 : int
634 1390673 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
635 : {
636 1390673 : const int *opmap = vect_get_operand_map (stmt);
637 1390673 : if (!opmap)
638 : return op;
639 21811 : for (int i = 1; i < 1 + opmap[0]; ++i)
640 21811 : if (opmap[i] == op)
641 12180 : return i - 1;
642 0 : gcc_unreachable ();
643 : }
644 :
645 : /* Helper class for mapping of GIMPLE operands to SLP children. */
646 : /* ??? Add vect_slp_child_index_for_operand here and amend opmaps
647 : with the full reverse mapping and indicating the position of the
648 : first commutative operand index, eliding the swap_p argument from
649 : vect_get_operand_map. Adjust all consumers. */
650 :
651 : struct slp_oprnds {
652 : slp_oprnds (stmt_vec_info);
653 : tree get_op_for_slp_child (stmt_vec_info, unsigned);
654 : const int *opmap;
655 : const unsigned int num_slp_children;
656 : };
657 :
658 4585778 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
659 4585778 : : opmap (vect_get_operand_map (stmt_info)),
660 4585778 : num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
661 : {
662 4585778 : }
663 :
664 : /* For SLP child number N get the corresponding tree operand from GIMPLE
665 : statement described by STMT_INFO. */
666 :
667 : tree
668 5121490 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
669 : {
670 5121490 : gcc_assert (n < num_slp_children);
671 5121490 : int opno = opmap ? opmap[n + 1] : (int) n;
672 5121490 : if (opno == GATHER_SCATTER_OFFSET)
673 0 : gcc_unreachable (); // TODO
674 5121490 : else if (opno < 0)
675 2522 : return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
676 : else
677 5118968 : return gimple_arg (stmt_info->stmt, opno);
678 : }
679 :
680 : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
681 : they are of a valid type and that they match the defs of the first stmt of
682 : the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
683 : by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP
684 : indicates swap is required for cond_expr stmts. Specifically, SWAP
685 : is 1 if STMT is cond and operands of comparison need to be swapped;
686 : SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
687 :
688 : If there was a fatal error return -1; if the error could be corrected by
689 : swapping operands of father node of this one, return 1; if everything is
690 : ok return 0. */
691 : static int
692 13065853 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
693 : bool *skip_args,
694 : vec<stmt_vec_info> stmts, unsigned stmt_num,
695 : vec<slp_oprnd_info> *oprnds_info)
696 : {
697 13065853 : stmt_vec_info stmt_info = stmts[stmt_num];
698 13065853 : tree oprnd;
699 13065853 : unsigned int i, number_of_oprnds;
700 13065853 : enum vect_def_type dt = vect_uninitialized_def;
701 13065853 : slp_oprnd_info oprnd_info;
702 13065853 : gather_scatter_info gs_info;
703 13065853 : unsigned int gs_op = -1u;
704 13065853 : unsigned int commutative_op = -1U;
705 13065853 : bool first = stmt_num == 0;
706 :
707 13065853 : if (!stmt_info)
708 : {
709 0 : for (auto oi : *oprnds_info)
710 : {
711 0 : oi->def_stmts.quick_push (NULL);
712 0 : oi->ops.quick_push (NULL_TREE);
713 : }
714 : return 0;
715 : }
716 :
717 13065853 : if (!is_a<gcall *> (stmt_info->stmt)
718 : && !is_a<gassign *> (stmt_info->stmt)
719 : && !is_a<gphi *> (stmt_info->stmt))
720 : return -1;
721 :
722 13065853 : number_of_oprnds = gimple_num_args (stmt_info->stmt);
723 13065853 : const int *map = vect_get_operand_map (stmt_info, swap);
724 13065853 : if (map)
725 80256 : number_of_oprnds = *map++;
726 13065853 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
727 : {
728 54584 : if (gimple_call_internal_p (stmt))
729 : {
730 33284 : internal_fn ifn = gimple_call_internal_fn (stmt);
731 33284 : commutative_op = first_commutative_argument (ifn);
732 33284 : if (internal_gather_scatter_fn_p (ifn))
733 : {
734 0 : vect_describe_gather_scatter_call
735 0 : (stmt_info,
736 0 : first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
737 0 : if (first)
738 0 : (*oprnds_info)[0]->first_gs_p = true;
739 : gs_op = 0;
740 : }
741 : }
742 : }
743 13011269 : else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
744 : {
745 15141366 : if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
746 8520170 : commutative_op = 0;
747 : }
748 :
749 13065853 : bool swapped = (swap != 0);
750 13065853 : bool backedge = false;
751 13065853 : enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
752 36135909 : for (i = 0; i < number_of_oprnds; i++)
753 : {
754 23071302 : oprnd_info = (*oprnds_info)[i];
755 23071302 : int opno = map ? map[i] : int (i);
756 23071302 : if (opno == GATHER_SCATTER_OFFSET)
757 : {
758 24053 : gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
759 24053 : if (!is_a <loop_vec_info> (vinfo)
760 24053 : || !vect_check_gather_scatter (stmt_info, vectype,
761 : as_a <loop_vec_info> (vinfo),
762 : first ? &oprnd_info->first_gs_info
763 : : &gs_info))
764 1246 : return -1;
765 :
766 24053 : if (first)
767 : {
768 23779 : oprnd_info->first_gs_p = true;
769 23779 : oprnd = oprnd_info->first_gs_info.offset;
770 : }
771 : else
772 : {
773 274 : gs_op = i;
774 274 : oprnd = gs_info.offset;
775 : }
776 : }
777 23047249 : else if (opno < 0)
778 3083 : oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
779 : else
780 : {
781 23044166 : oprnd = gimple_arg (stmt_info->stmt, opno);
782 23044166 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
783 : {
784 1269600 : edge e = gimple_phi_arg_edge (stmt, opno);
785 2539200 : backedge = (is_a <bb_vec_info> (vinfo)
786 1962472 : ? e->flags & EDGE_DFS_BACK
787 692872 : : dominated_by_p (CDI_DOMINATORS, e->src,
788 692872 : gimple_bb (stmt_info->stmt)));
789 : }
790 : }
791 :
792 23071302 : stmt_vec_info def_stmt_info;
793 23071302 : if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
794 : {
795 991 : if (dump_enabled_p ())
796 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
797 : "Build SLP failed: can't analyze def for %T\n",
798 : oprnd);
799 :
800 991 : return -1;
801 : }
802 :
803 23070311 : if (skip_args[i])
804 : {
805 556021 : oprnd_info->def_stmts.quick_push (NULL);
806 556021 : oprnd_info->ops.quick_push (NULL_TREE);
807 556021 : oprnd_info->first_dt = vect_uninitialized_def;
808 556021 : continue;
809 : }
810 :
811 22514290 : oprnd_info->def_stmts.quick_push (def_stmt_info);
812 22514290 : oprnd_info->ops.quick_push (oprnd);
813 :
814 22514290 : if (def_stmt_info
815 22514290 : && is_pattern_stmt_p (def_stmt_info))
816 : {
817 442937 : if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
818 : != def_stmt_info)
819 317644 : oprnd_info->any_pattern = true;
820 : else
821 : /* If we promote this to external use the original stmt def. */
822 125293 : oprnd_info->ops.last ()
823 250586 : = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
824 : }
825 :
826 : /* If there's a extern def on a backedge make sure we can
827 : code-generate at the region start.
828 : ??? This is another case that could be fixed by adjusting
829 : how we split the function but at the moment we'd have conflicting
830 : goals there. */
831 22514290 : if (backedge
832 173389 : && dts[i] == vect_external_def
833 276 : && is_a <bb_vec_info> (vinfo)
834 276 : && TREE_CODE (oprnd) == SSA_NAME
835 255 : && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
836 22514545 : && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
837 255 : gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
838 : {
839 255 : if (dump_enabled_p ())
840 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
841 : "Build SLP failed: extern def %T only defined "
842 : "on backedge\n", oprnd);
843 255 : return -1;
844 : }
845 :
846 22514035 : if (first)
847 : {
848 5128045 : tree type = TREE_TYPE (oprnd);
849 5128045 : dt = dts[i];
850 :
851 : /* For the swapping logic below force vect_reduction_def
852 : for the reduction op in a SLP reduction group. */
853 5128045 : if (!STMT_VINFO_DATA_REF (stmt_info)
854 3939521 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
855 5242 : && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
856 5130626 : && def_stmt_info)
857 2581 : dts[i] = dt = vect_reduction_def;
858 :
859 : /* Check the types of the definition. */
860 5128045 : switch (dt)
861 : {
862 5128045 : case vect_external_def:
863 5128045 : case vect_constant_def:
864 5128045 : case vect_internal_def:
865 5128045 : case vect_reduction_def:
866 5128045 : case vect_double_reduction_def:
867 5128045 : case vect_induction_def:
868 5128045 : case vect_nested_cycle:
869 5128045 : case vect_first_order_recurrence:
870 5128045 : break;
871 :
872 0 : default:
873 : /* FORNOW: Not supported. */
874 0 : if (dump_enabled_p ())
875 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
876 : "Build SLP failed: illegal type of def %T\n",
877 : oprnd);
878 0 : return -1;
879 : }
880 :
881 5128045 : oprnd_info->first_dt = dt;
882 5128045 : oprnd_info->first_op_type = type;
883 : }
884 : }
885 13064607 : if (first)
886 : return 0;
887 :
888 : /* Now match the operand definition types to that of the first stmt. */
889 26619537 : for (i = 0; i < number_of_oprnds;)
890 : {
891 17380357 : if (skip_args[i])
892 : {
893 44403 : ++i;
894 44403 : continue;
895 : }
896 :
897 17335954 : oprnd_info = (*oprnds_info)[i];
898 17335954 : dt = dts[i];
899 17335954 : stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
900 17335954 : oprnd = oprnd_info->ops[stmt_num];
901 17335954 : tree type = TREE_TYPE (oprnd);
902 :
903 17335954 : if (!types_compatible_p (oprnd_info->first_op_type, type))
904 : {
905 108517 : if (dump_enabled_p ())
906 93 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
907 : "Build SLP failed: different operand types\n");
908 108517 : return 1;
909 : }
910 :
911 17227437 : if ((gs_op == i) != oprnd_info->first_gs_p)
912 : {
913 0 : if (dump_enabled_p ())
914 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
915 : "Build SLP failed: mixed gather and non-gather\n");
916 0 : return 1;
917 : }
918 17227437 : else if (gs_op == i)
919 : {
920 242 : if (!operand_equal_p (oprnd_info->first_gs_info.base,
921 242 : gs_info.base))
922 : {
923 16 : if (dump_enabled_p ())
924 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
925 : "Build SLP failed: different gather base\n");
926 16 : return 1;
927 : }
928 226 : if (oprnd_info->first_gs_info.scale != gs_info.scale)
929 : {
930 8 : if (dump_enabled_p ())
931 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
932 : "Build SLP failed: different gather scale\n");
933 8 : return 1;
934 : }
935 : }
936 :
937 : /* Not first stmt of the group, check that the def-stmt/s match
938 : the def-stmt/s of the first stmt. Allow different definition
939 : types for reduction chains: the first stmt must be a
940 : vect_reduction_def (a phi node), and the rest
941 : end in the reduction chain. */
942 17227413 : if ((!vect_def_types_match (oprnd_info->first_dt, dt)
943 305987 : && !(oprnd_info->first_dt == vect_reduction_def
944 4806 : && !STMT_VINFO_DATA_REF (stmt_info)
945 4806 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
946 4780 : && def_stmt_info
947 4778 : && !STMT_VINFO_DATA_REF (def_stmt_info)
948 4778 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
949 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
950 16926204 : || (!STMT_VINFO_DATA_REF (stmt_info)
951 15599889 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
952 9961 : && ((!def_stmt_info
953 9765 : || STMT_VINFO_DATA_REF (def_stmt_info)
954 17987 : || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
955 : != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
956 9961 : != (oprnd_info->first_dt != vect_reduction_def))))
957 : {
958 : /* Try swapping operands if we got a mismatch. For BB
959 : vectorization only in case it will clearly improve things. */
960 303932 : if (i == commutative_op && !swapped
961 301209 : && (!is_a <bb_vec_info> (vinfo)
962 5261 : || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
963 5261 : dts[i+1])
964 1486 : && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
965 : || vect_def_types_match
966 224 : ((*oprnds_info)[i+1]->first_dt, dts[i])))))
967 : {
968 2723 : if (dump_enabled_p ())
969 153 : dump_printf_loc (MSG_NOTE, vect_location,
970 : "trying swapped operands\n");
971 2723 : std::swap (dts[i], dts[i+1]);
972 2723 : std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
973 2723 : (*oprnds_info)[i+1]->def_stmts[stmt_num]);
974 2723 : std::swap ((*oprnds_info)[i]->ops[stmt_num],
975 2723 : (*oprnds_info)[i+1]->ops[stmt_num]);
976 : /* After swapping some operands we lost track whether an
977 : operand has any pattern defs so be conservative here. */
978 2723 : if ((*oprnds_info)[i]->any_pattern
979 2723 : || (*oprnds_info)[i+1]->any_pattern)
980 36 : (*oprnds_info)[i]->any_pattern
981 18 : = (*oprnds_info)[i+1]->any_pattern = true;
982 2723 : swapped = true;
983 2723 : continue;
984 : }
985 :
986 298486 : if (is_a <bb_vec_info> (vinfo)
987 282946 : && !oprnd_info->any_pattern
988 581181 : && number_of_oprnds > 1)
989 : {
990 : /* Now for commutative ops we should see whether we can
991 : make the other operand matching. */
992 107734 : if (dump_enabled_p ())
993 251 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
994 : "treating operand as external\n");
995 107734 : oprnd_info->first_dt = dt = vect_external_def;
996 : }
997 : else
998 : {
999 190752 : if (dump_enabled_p ())
1000 411 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1001 : "Build SLP failed: different types\n");
1002 190752 : return 1;
1003 : }
1004 : }
1005 :
1006 : /* Make sure to demote the overall operand to external. */
1007 17033938 : if (dt == vect_external_def)
1008 348960 : oprnd_info->first_dt = vect_external_def;
1009 : /* For a SLP reduction chain we want to duplicate the reduction to
1010 : each of the chain members. That gets us a sane SLP graph (still
1011 : the stmts are not 100% correct wrt the initial values). */
1012 16684978 : else if ((dt == vect_internal_def
1013 16684978 : || dt == vect_reduction_def)
1014 15717473 : && oprnd_info->first_dt == vect_reduction_def
1015 101220 : && !STMT_VINFO_DATA_REF (stmt_info)
1016 101220 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
1017 4778 : && !STMT_VINFO_DATA_REF (def_stmt_info)
1018 16689756 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
1019 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
1020 : {
1021 4778 : oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
1022 4778 : oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
1023 : }
1024 :
1025 17033938 : ++i;
1026 : }
1027 :
1028 : /* Swap operands. */
1029 9239180 : if (swapped)
1030 : {
1031 43169 : if (dump_enabled_p ())
1032 457 : dump_printf_loc (MSG_NOTE, vect_location,
1033 : "swapped operands to match def types in %G",
1034 : stmt_info->stmt);
1035 : }
1036 :
1037 : return 0;
1038 : }
1039 :
1040 : /* Return true if call statements CALL1 and CALL2 are similar enough
1041 : to be combined into the same SLP group. */
1042 :
1043 : bool
1044 70778 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
1045 : {
1046 70778 : unsigned int nargs = gimple_call_num_args (call1);
1047 70778 : if (nargs != gimple_call_num_args (call2))
1048 : return false;
1049 :
1050 64417 : auto cfn1 = gimple_call_combined_fn (call1);
1051 64417 : auto cfn2 = gimple_call_combined_fn (call2);
1052 64417 : if (cfn1 != cfn2
1053 2 : && (!allow_two_operators
1054 2 : || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
1055 2 : && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
1056 : return false;
1057 :
1058 64417 : if (gimple_call_internal_p (call1))
1059 : {
1060 7241 : if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
1061 7241 : TREE_TYPE (gimple_call_lhs (call2))))
1062 : return false;
1063 14878 : for (unsigned int i = 0; i < nargs; ++i)
1064 7637 : if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
1065 7637 : TREE_TYPE (gimple_call_arg (call2, i))))
1066 : return false;
1067 : }
1068 : else
1069 : {
1070 57176 : if (!operand_equal_p (gimple_call_fn (call1),
1071 57176 : gimple_call_fn (call2), 0))
1072 : return false;
1073 :
1074 42282 : if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
1075 : return false;
1076 : }
1077 :
1078 : /* Check that any unvectorized arguments are equal. */
1079 21335 : if (const int *map = vect_get_operand_map (call1, false, false))
1080 : {
1081 15 : unsigned int nkept = *map++;
1082 15 : unsigned int mapi = 0;
1083 57 : for (unsigned int i = 0; i < nargs; ++i)
1084 42 : if (mapi < nkept && map[mapi] == int (i))
1085 27 : mapi += 1;
1086 15 : else if (!operand_equal_p (gimple_call_arg (call1, i),
1087 15 : gimple_call_arg (call2, i)))
1088 : return false;
1089 : }
1090 :
1091 : return true;
1092 : }
1093 :
1094 : /* A subroutine of vect_build_slp_tree for checking VECTYPE, which is the
1095 : caller's attempt to find the vector type in STMT_INFO with the narrowest
1096 : element type. Return true if VECTYPE is nonnull and if it is valid
1097 : for STMT_INFO. When returning true, update MAX_NUNITS to reflect the
1098 : number of units in VECTYPE. GROUP_SIZE and MAX_NUNITS are as for
1099 : vect_build_slp_tree. */
1100 :
1101 : static bool
1102 5892385 : vect_record_max_nunits (vec_info *vinfo, stmt_vec_info stmt_info,
1103 : unsigned int group_size,
1104 : tree vectype, poly_uint64 *max_nunits)
1105 : {
1106 5892385 : if (!vectype)
1107 : {
1108 4028 : if (dump_enabled_p ())
1109 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1110 : "Build SLP failed: unsupported data-type in %G\n",
1111 : stmt_info->stmt);
1112 : /* Fatal mismatch. */
1113 4028 : return false;
1114 : }
1115 :
1116 : /* If populating the vector type requires unrolling then fail
1117 : before adjusting *max_nunits for basic-block vectorization. */
1118 5888357 : if (is_a <bb_vec_info> (vinfo)
1119 5888357 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
1120 : {
1121 215950 : if (dump_enabled_p ())
1122 134 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1123 : "Build SLP failed: unrolling required "
1124 : "in basic block SLP\n");
1125 : /* Fatal mismatch. */
1126 215950 : return false;
1127 : }
1128 :
1129 : /* In case of multiple types we need to detect the smallest type. */
1130 5672407 : vect_update_max_nunits (max_nunits, vectype);
1131 5672407 : return true;
1132 : }
1133 :
1134 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1135 : permutation or are of unsupported types of operation. Return
1136 : true if they are, otherwise return false and indicate in *MATCHES
1137 : which stmts are not isomorphic to the first one. If MATCHES[0]
1138 : is false then this indicates the comparison could not be
1139 : carried out or the stmts will never be vectorized by SLP.
1140 :
1141 : Note COND_EXPR is possibly isomorphic to another one after swapping its
1142 : operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
1143 : the first stmt by swapping the two operands of comparison; set SWAP[i]
1144 : to 2 if stmt I is isormorphic to the first stmt by inverting the code
1145 : of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
1146 : to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
1147 :
1148 : static bool
1149 6204224 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
1150 : vec<stmt_vec_info> stmts,
1151 : poly_uint64 *max_nunits, bool *matches,
1152 : bool *two_operators, tree *node_vectype)
1153 : {
1154 6204224 : unsigned int group_size = stmts.length ();
1155 6204224 : unsigned int i;
1156 6204224 : stmt_vec_info first_stmt_info = stmts[0];
1157 6204224 : code_helper first_stmt_code = ERROR_MARK;
1158 6204224 : code_helper alt_stmt_code = ERROR_MARK;
1159 6204224 : code_helper first_cond_code = ERROR_MARK;
1160 6204224 : bool need_same_oprnds = false;
1161 6204224 : tree first_lhs = NULL_TREE;
1162 6204224 : tree first_op1 = NULL_TREE;
1163 6204224 : stmt_vec_info first_load = NULL, prev_first_load = NULL;
1164 6204224 : bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
1165 6204224 : bool first_stmt_phi_p = false;
1166 6204224 : int first_reduc_idx = -1;
1167 6204224 : bool maybe_soft_fail = false;
1168 6204224 : tree soft_fail_nunits_vectype = NULL_TREE;
1169 :
1170 6204224 : tree vectype, nunits_vectype;
1171 6204224 : if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
1172 : &nunits_vectype, group_size))
1173 : {
1174 : /* Fatal mismatch. */
1175 261204 : matches[0] = false;
1176 261204 : return false;
1177 : }
1178 5943020 : if (is_a <bb_vec_info> (vinfo)
1179 5943020 : && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
1180 : {
1181 366113 : if (dump_enabled_p ())
1182 287 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1183 : "Build SLP failed: not using single lane "
1184 : "vector type %T\n", vectype);
1185 366113 : matches[0] = false;
1186 366113 : return false;
1187 : }
1188 : /* Record nunits required but continue analysis, producing matches[]
1189 : as if nunits was not an issue. This allows splitting of groups
1190 : to happen. */
1191 5576907 : if (nunits_vectype
1192 5576907 : && !vect_record_max_nunits (vinfo, first_stmt_info, group_size,
1193 : nunits_vectype, max_nunits))
1194 : {
1195 215950 : gcc_assert (is_a <bb_vec_info> (vinfo));
1196 215950 : maybe_soft_fail = true;
1197 215950 : soft_fail_nunits_vectype = nunits_vectype;
1198 : }
1199 :
1200 5576907 : gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
1201 5576907 : *node_vectype = vectype;
1202 :
1203 : /* For every stmt in NODE find its def stmt/s. */
1204 5576907 : stmt_vec_info stmt_info;
1205 23482766 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
1206 : {
1207 18083093 : bool ldst_p = false;
1208 18083093 : bool ldst_masklen_p = false;
1209 18083093 : bool phi_p = false;
1210 18083093 : code_helper rhs_code = ERROR_MARK;
1211 :
1212 18083093 : swap[i] = 0;
1213 18083093 : matches[i] = false;
1214 18083093 : if (!stmt_info)
1215 : {
1216 41079 : matches[i] = true;
1217 17946938 : continue;
1218 : }
1219 :
1220 18042014 : gimple *stmt = stmt_info->stmt;
1221 18042014 : if (dump_enabled_p ())
1222 226146 : dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
1223 :
1224 : /* Fail to vectorize statements marked as unvectorizable, throw
1225 : or are volatile. */
1226 18042014 : if (!STMT_VINFO_VECTORIZABLE (stmt_info)
1227 17790454 : || stmt_can_throw_internal (cfun, stmt)
1228 34967655 : || gimple_has_volatile_ops (stmt))
1229 : {
1230 258512 : if (dump_enabled_p ())
1231 232 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1232 : "Build SLP failed: unvectorizable statement %G",
1233 : stmt);
1234 : /* ??? For BB vectorization we want to commutate operands in a way
1235 : to shuffle all unvectorizable defs into one operand and have
1236 : the other still vectorized. The following doesn't reliably
1237 : work for this though but it's the easiest we can do here. */
1238 258512 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1239 106654 : continue;
1240 : /* Fatal mismatch. */
1241 151858 : matches[0] = false;
1242 151858 : return false;
1243 : }
1244 :
1245 17783502 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
1246 17783502 : tree lhs = gimple_get_lhs (stmt);
1247 17783502 : if (lhs == NULL_TREE && !call_stmt)
1248 : {
1249 36 : if (dump_enabled_p ())
1250 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1251 : "Build SLP failed: not GIMPLE_ASSIGN nor "
1252 : "GIMPLE_CALL %G", stmt);
1253 36 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1254 36 : continue;
1255 : /* Fatal mismatch. */
1256 0 : matches[0] = false;
1257 0 : return false;
1258 : }
1259 :
1260 17783466 : if (call_stmt)
1261 : {
1262 174420 : combined_fn cfn = gimple_call_combined_fn (call_stmt);
1263 174420 : if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
1264 59936 : rhs_code = cfn;
1265 : else
1266 : rhs_code = CALL_EXPR;
1267 :
1268 174420 : if (cfn == CFN_GATHER_LOAD
1269 174420 : || cfn == CFN_SCATTER_STORE)
1270 : ldst_p = true;
1271 : else if (cfn == CFN_MASK_LOAD
1272 : || cfn == CFN_MASK_GATHER_LOAD
1273 : || cfn == CFN_MASK_LEN_GATHER_LOAD
1274 : || cfn == CFN_MASK_SCATTER_STORE
1275 : || cfn == CFN_MASK_LEN_SCATTER_STORE)
1276 : {
1277 : ldst_p = true;
1278 : ldst_masklen_p = true;
1279 : }
1280 : else if (cfn == CFN_MASK_STORE)
1281 : {
1282 : ldst_p = true;
1283 : ldst_masklen_p = true;
1284 : rhs_code = CFN_MASK_STORE;
1285 : }
1286 : else if (cfn == CFN_GOMP_SIMD_LANE)
1287 : ;
1288 162546 : else if ((cfn != CFN_LAST
1289 : && cfn != CFN_MASK_CALL
1290 48062 : && internal_fn_p (cfn)
1291 37616 : && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
1292 162463 : || gimple_call_tail_p (call_stmt)
1293 162463 : || gimple_call_noreturn_p (call_stmt)
1294 325009 : || gimple_call_chain (call_stmt))
1295 : {
1296 443 : if (dump_enabled_p ())
1297 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1298 : "Build SLP failed: unsupported call type %G",
1299 : (gimple *) call_stmt);
1300 443 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1301 72 : continue;
1302 : /* Fatal mismatch. */
1303 371 : matches[0] = false;
1304 371 : return false;
1305 : }
1306 : }
1307 17609046 : else if (gimple_code (stmt) == GIMPLE_PHI)
1308 : {
1309 : rhs_code = ERROR_MARK;
1310 : phi_p = true;
1311 : }
1312 : else
1313 : {
1314 16744233 : rhs_code = gimple_assign_rhs_code (stmt);
1315 16744233 : ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
1316 : }
1317 :
1318 : /* Check the operation. */
1319 17783023 : if (i == 0)
1320 : {
1321 5424678 : first_lhs = lhs;
1322 5424678 : first_stmt_code = rhs_code;
1323 5424678 : first_stmt_ldst_p = ldst_p;
1324 5424678 : first_stmt_ldst_masklen_p = ldst_masklen_p;
1325 5424678 : first_stmt_phi_p = phi_p;
1326 5424678 : first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
1327 :
1328 : /* Shift arguments should be equal in all the packed stmts for a
1329 : vector shift with scalar shift operand. */
1330 5424678 : if (rhs_code == LSHIFT_EXPR
1331 5353143 : || rhs_code == RSHIFT_EXPR
1332 5273434 : || rhs_code == LROTATE_EXPR
1333 10697998 : || rhs_code == RROTATE_EXPR)
1334 : {
1335 : /* First see if we have a vector/vector shift. */
1336 151836 : if (!directly_supported_p (rhs_code, vectype, optab_vector))
1337 : {
1338 : /* No vector/vector shift, arrange for a vector/scalar
1339 : SLP layout. */
1340 139645 : need_same_oprnds = true;
1341 139645 : first_op1 = gimple_assign_rhs2 (stmt);
1342 : }
1343 : }
1344 5272842 : else if (rhs_code == WIDEN_LSHIFT_EXPR)
1345 : {
1346 0 : need_same_oprnds = true;
1347 0 : first_op1 = gimple_assign_rhs2 (stmt);
1348 : }
1349 5272842 : else if (!ldst_p
1350 5272842 : && rhs_code == BIT_FIELD_REF)
1351 : {
1352 8900 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
1353 8900 : if (!is_a <bb_vec_info> (vinfo)
1354 8774 : || TREE_CODE (vec) != SSA_NAME
1355 : /* When the element types are not compatible we pun the
1356 : source to the target vectype which requires equal size. */
1357 17662 : || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
1358 7957 : || !types_compatible_p (TREE_TYPE (vectype),
1359 7957 : TREE_TYPE (TREE_TYPE (vec))))
1360 1124 : && !operand_equal_p (TYPE_SIZE (vectype),
1361 1124 : TYPE_SIZE (TREE_TYPE (vec)))))
1362 : {
1363 853 : if (dump_enabled_p ())
1364 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1365 : "Build SLP failed: "
1366 : "BIT_FIELD_REF not supported\n");
1367 : /* Fatal mismatch. */
1368 853 : matches[0] = false;
1369 853 : return false;
1370 : }
1371 : }
1372 5263942 : else if (rhs_code == CFN_DIV_POW2)
1373 : {
1374 0 : need_same_oprnds = true;
1375 0 : first_op1 = gimple_call_arg (call_stmt, 1);
1376 : }
1377 5263942 : else if (rhs_code == CFN_GOMP_SIMD_LANE)
1378 : {
1379 3645 : need_same_oprnds = true;
1380 3645 : first_op1 = gimple_call_arg (call_stmt, 1);
1381 : }
1382 : }
1383 : else
1384 : {
1385 12358345 : int comm_arg;
1386 12358723 : if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1387 : /* For SLP reduction groups the index isn't necessarily
1388 : uniform but only that of the first stmt matters. */
1389 2340 : && !(first_reduc_idx != -1
1390 2340 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1391 2340 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
1392 12358345 : && !(first_reduc_idx != -1
1393 1049 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1394 1049 : && (comm_arg = first_commutative_argument
1395 1049 : (rhs_code, TREE_TYPE (lhs))) >= 0
1396 : && (first_reduc_idx
1397 815 : == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
1398 : {
1399 378 : if (dump_enabled_p ())
1400 : {
1401 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1402 : "Build SLP failed: different reduc_idx "
1403 : "%d instead of %d in %G",
1404 : STMT_VINFO_REDUC_IDX (stmt_info),
1405 : first_reduc_idx, stmt);
1406 : }
1407 : /* Mismatch. */
1408 378 : continue;
1409 : }
1410 12357967 : if (!ldst_p
1411 9727377 : && first_stmt_code != rhs_code
1412 13884642 : && alt_stmt_code == ERROR_MARK)
1413 : alt_stmt_code = rhs_code;
1414 13862473 : if ((!ldst_p
1415 9727377 : && first_stmt_code != rhs_code
1416 1526675 : && (first_stmt_code != IMAGPART_EXPR
1417 220 : || rhs_code != REALPART_EXPR)
1418 1526654 : && (first_stmt_code != REALPART_EXPR
1419 743 : || rhs_code != IMAGPART_EXPR)
1420 : /* Handle mismatches in plus/minus by computing both
1421 : and merging the results. */
1422 1526643 : && !((((first_stmt_code == PLUS_EXPR
1423 1407661 : || first_stmt_code == MINUS_EXPR)
1424 149109 : && (alt_stmt_code == PLUS_EXPR
1425 139272 : || alt_stmt_code == MINUS_EXPR))
1426 1495330 : || ((first_stmt_code == CFN_FMA
1427 1495328 : || first_stmt_code == CFN_FMS)
1428 2 : && (alt_stmt_code == CFN_FMA
1429 2 : || alt_stmt_code == CFN_FMS)))
1430 31315 : && rhs_code == alt_stmt_code)
1431 1539118 : && !(first_stmt_code.is_tree_code ()
1432 1397794 : && rhs_code.is_tree_code ()
1433 1281345 : && (TREE_CODE_CLASS (tree_code (first_stmt_code))
1434 : == tcc_comparison)
1435 140285 : && (swap_tree_comparison (tree_code (first_stmt_code))
1436 140285 : == tree_code (rhs_code))
1437 : && (first_reduc_idx == -1
1438 0 : || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
1439 : || (ldst_p
1440 5261180 : && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1441 2630590 : != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
1442 : || (ldst_p
1443 2578451 : && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1444 2578451 : != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
1445 10853625 : || first_stmt_ldst_p != ldst_p
1446 10853469 : || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
1447 23211428 : || first_stmt_phi_p != phi_p)
1448 : {
1449 1504506 : if (dump_enabled_p ())
1450 : {
1451 3303 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1452 : "Build SLP failed: different operation "
1453 : "in stmt %G", stmt);
1454 3303 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1455 : "original stmt %G", first_stmt_info->stmt);
1456 : }
1457 : /* Mismatch. */
1458 1504506 : continue;
1459 : }
1460 :
1461 10870268 : if (!ldst_p
1462 8275145 : && first_stmt_code == BIT_FIELD_REF
1463 10880026 : && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
1464 26565 : != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
1465 : {
1466 16807 : if (dump_enabled_p ())
1467 76 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1468 : "Build SLP failed: different BIT_FIELD_REF "
1469 : "arguments in %G", stmt);
1470 : /* Mismatch. */
1471 16807 : continue;
1472 : }
1473 :
1474 10836654 : if (call_stmt
1475 71491 : && first_stmt_code != CFN_MASK_LOAD
1476 10907743 : && first_stmt_code != CFN_MASK_STORE)
1477 : {
1478 70778 : if (!is_a <gcall *> (stmts[0]->stmt)
1479 70778 : || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
1480 : call_stmt, true))
1481 : {
1482 49443 : if (dump_enabled_p ())
1483 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1484 : "Build SLP failed: different calls in %G",
1485 : stmt);
1486 : /* Mismatch. */
1487 49443 : continue;
1488 : }
1489 : }
1490 :
1491 10578132 : if ((phi_p || gimple_could_trap_p (stmt_info->stmt))
1492 11667209 : && (gimple_bb (first_stmt_info->stmt)
1493 1089077 : != gimple_bb (stmt_info->stmt)))
1494 : {
1495 47629 : if (dump_enabled_p ())
1496 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1497 : "Build SLP failed: different BB for PHI "
1498 : "or possibly trapping operation in %G", stmt);
1499 : /* Mismatch. */
1500 47629 : continue;
1501 : }
1502 :
1503 10739582 : if (need_same_oprnds)
1504 : {
1505 68836 : tree other_op1 = gimple_arg (stmt, 1);
1506 68836 : if (!operand_equal_p (first_op1, other_op1, 0))
1507 : {
1508 8943 : if (dump_enabled_p ())
1509 133 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1510 : "Build SLP failed: different shift "
1511 : "arguments in %G", stmt);
1512 : /* Mismatch. */
1513 8943 : continue;
1514 : }
1515 : }
1516 :
1517 10731376 : if (first_lhs
1518 10730639 : && lhs
1519 10730639 : && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
1520 : {
1521 737 : if (dump_enabled_p ())
1522 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1523 : "Build SLP failed: different vector type "
1524 : "in %G", stmt);
1525 : /* Mismatch. */
1526 737 : continue;
1527 : }
1528 : }
1529 :
1530 : /* Grouped store or load. */
1531 16153727 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1532 : {
1533 3985809 : gcc_assert (ldst_p);
1534 3985809 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
1535 : {
1536 : /* Store. */
1537 3081891 : gcc_assert (rhs_code == CFN_MASK_STORE
1538 : || REFERENCE_CLASS_P (lhs)
1539 : || DECL_P (lhs));
1540 : }
1541 : else
1542 : {
1543 : /* Load. */
1544 903918 : first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
1545 903918 : if (prev_first_load)
1546 : {
1547 : /* Check that there are no loads from different interleaving
1548 : chains in the same node. */
1549 416681 : if (prev_first_load != first_load)
1550 : {
1551 57806 : if (dump_enabled_p ())
1552 2178 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
1553 : vect_location,
1554 : "Build SLP failed: different "
1555 : "interleaving chains in one node %G",
1556 : stmt);
1557 : /* Mismatch. */
1558 57806 : continue;
1559 : }
1560 : }
1561 : else
1562 : prev_first_load = first_load;
1563 : }
1564 : }
1565 : /* Non-grouped store or load. */
1566 12167918 : else if (ldst_p)
1567 : {
1568 918030 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1569 642283 : && rhs_code != CFN_GATHER_LOAD
1570 : && rhs_code != CFN_MASK_GATHER_LOAD
1571 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1572 : && rhs_code != CFN_SCATTER_STORE
1573 : && rhs_code != CFN_MASK_SCATTER_STORE
1574 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1575 642283 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1576 : /* Not grouped loads are handled as externals for BB
1577 : vectorization. For loop vectorization we can handle
1578 : splats the same we handle single element interleaving.
1579 : Likewise we can handle a collection of invariant refs. */
1580 1540489 : && (is_a <bb_vec_info> (vinfo)
1581 622459 : || (stmt_info != first_stmt_info
1582 68115 : && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
1583 241 : && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
1584 : (first_stmt_info)))))))
1585 : {
1586 : /* Not grouped load. */
1587 67633 : if (dump_enabled_p ())
1588 145 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1589 : "Build SLP failed: not grouped load %G", stmt);
1590 :
1591 67633 : if (i != 0)
1592 67633 : continue;
1593 : /* Fatal mismatch. */
1594 0 : matches[0] = false;
1595 0 : return false;
1596 : }
1597 : }
1598 : /* Not memory operation. */
1599 : else
1600 : {
1601 11249888 : if (!phi_p
1602 10546107 : && rhs_code.is_tree_code ()
1603 10496503 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
1604 1809924 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
1605 1124803 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
1606 1054907 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
1607 89492 : && rhs_code != VIEW_CONVERT_EXPR
1608 : && rhs_code != CALL_EXPR
1609 : && rhs_code != BIT_FIELD_REF
1610 11249888 : && rhs_code != SSA_NAME)
1611 : {
1612 24152 : if (dump_enabled_p ())
1613 17 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1614 : "Build SLP failed: operation unsupported %G",
1615 : stmt);
1616 24152 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1617 0 : continue;
1618 : /* Fatal mismatch. */
1619 24152 : matches[0] = false;
1620 24152 : return false;
1621 : }
1622 :
1623 11225736 : if (rhs_code == COND_EXPR)
1624 : {
1625 67045 : tree cond_expr = gimple_assign_rhs1 (stmt);
1626 67045 : enum tree_code cond_code = TREE_CODE (cond_expr);
1627 67045 : enum tree_code swap_code = ERROR_MARK;
1628 67045 : enum tree_code invert_code = ERROR_MARK;
1629 :
1630 67045 : if (i == 0)
1631 54789 : first_cond_code = TREE_CODE (cond_expr);
1632 12256 : else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
1633 : {
1634 0 : bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
1635 0 : swap_code = swap_tree_comparison (cond_code);
1636 0 : invert_code = invert_tree_comparison (cond_code, honor_nans);
1637 : }
1638 :
1639 67045 : if (first_cond_code == cond_code)
1640 : ;
1641 : /* Isomorphic can be achieved by swapping. */
1642 0 : else if (first_cond_code == swap_code)
1643 0 : swap[i] = 1;
1644 : /* Isomorphic can be achieved by inverting. */
1645 0 : else if (first_cond_code == invert_code)
1646 0 : swap[i] = 2;
1647 : else
1648 : {
1649 0 : if (dump_enabled_p ())
1650 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1651 : "Build SLP failed: different"
1652 : " operation %G", stmt);
1653 : /* Mismatch. */
1654 0 : continue;
1655 : }
1656 : }
1657 :
1658 11225736 : if (i != 0
1659 8157325 : && first_stmt_code != rhs_code
1660 74455 : && first_stmt_code.is_tree_code ()
1661 74453 : && rhs_code.is_tree_code ()
1662 74453 : && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
1663 11269203 : && (swap_tree_comparison ((tree_code)first_stmt_code)
1664 43467 : == (tree_code)rhs_code))
1665 43467 : swap[i] = 1;
1666 :
1667 11225736 : if (i != 0
1668 8157325 : && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1669 1692 : && first_reduc_idx != -1
1670 1692 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1671 1692 : && rhs_code.is_tree_code ()
1672 1684 : && commutative_tree_code (tree_code (rhs_code))
1673 11227418 : && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
1674 1682 : swap[i] = 1;
1675 : }
1676 :
1677 16004136 : matches[i] = true;
1678 : }
1679 :
1680 21395004 : for (i = 0; i < group_size; ++i)
1681 16779999 : if (!matches[i])
1682 : return false;
1683 :
1684 : /* If we allowed a two-operation SLP node verify the target can cope
1685 : with the permute we are going to use. */
1686 4615005 : if (alt_stmt_code != ERROR_MARK
1687 4615005 : && (!alt_stmt_code.is_tree_code ()
1688 57550 : || (TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_reference
1689 57550 : && TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_comparison)))
1690 : {
1691 16291 : *two_operators = true;
1692 : }
1693 :
1694 4615005 : if (maybe_soft_fail)
1695 : {
1696 161545 : unsigned HOST_WIDE_INT const_nunits;
1697 161545 : if (!TYPE_VECTOR_SUBPARTS
1698 161545 : (soft_fail_nunits_vectype).is_constant (&const_nunits)
1699 161545 : || const_nunits > group_size)
1700 0 : matches[0] = false;
1701 : else
1702 : {
1703 : /* With constant vector elements simulate a mismatch at the
1704 : point we need to split. */
1705 161545 : unsigned tail = group_size & (const_nunits - 1);
1706 161545 : memset (&matches[group_size - tail], 0, sizeof (bool) * tail);
1707 : }
1708 161545 : return false;
1709 : }
1710 :
1711 : return true;
1712 : }
1713 :
1714 : /* Traits for the hash_set to record failed SLP builds for a stmt set.
1715 : Note we never remove apart from at destruction time so we do not
1716 : need a special value for deleted that differs from empty. */
1717 : struct bst_traits
1718 : {
1719 : typedef vec <stmt_vec_info> value_type;
1720 : typedef vec <stmt_vec_info> compare_type;
1721 : static inline hashval_t hash (value_type);
1722 : static inline bool equal (value_type existing, value_type candidate);
1723 511397681 : static inline bool is_empty (value_type x) { return !x.exists (); }
1724 115348305 : static inline bool is_deleted (value_type x) { return !x.exists (); }
1725 : static const bool empty_zero_p = true;
1726 0 : static inline void mark_empty (value_type &x) { x.release (); }
1727 : static inline void mark_deleted (value_type &x) { x.release (); }
1728 9836855 : static inline void remove (value_type &x) { x.release (); }
1729 : };
1730 : inline hashval_t
1731 100338193 : bst_traits::hash (value_type x)
1732 : {
1733 100338193 : inchash::hash h;
1734 446159274 : for (unsigned i = 0; i < x.length (); ++i)
1735 345821081 : h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
1736 100338193 : return h.end ();
1737 : }
1738 : inline bool
1739 88003866 : bst_traits::equal (value_type existing, value_type candidate)
1740 : {
1741 264011598 : if (existing.length () != candidate.length ())
1742 : return false;
1743 87948899 : for (unsigned i = 0; i < existing.length (); ++i)
1744 83397523 : if (existing[i] != candidate[i])
1745 : return false;
1746 : return true;
1747 : }
1748 :
1749 : typedef hash_map <vec <stmt_vec_info>, slp_tree,
1750 : simple_hashmap_traits <bst_traits, slp_tree> >
1751 : scalar_stmts_to_slp_tree_map_t;
1752 :
1753 : /* Release BST_MAP. */
1754 :
1755 : static void
1756 1888402 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
1757 : {
1758 : /* The map keeps a reference on SLP nodes built, release that. */
1759 11725257 : for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
1760 21562112 : it != bst_map->end (); ++it)
1761 9836855 : if ((*it).second)
1762 9836855 : vect_free_slp_tree ((*it).second);
1763 1888402 : delete bst_map;
1764 1888402 : }
1765 :
1766 : /* ??? This was std::pair<std::pair<tree_code, vect_def_type>, tree>
1767 : but then vec::insert does memmove and that's not compatible with
1768 : std::pair. */
1769 : struct chain_op_t
1770 : {
1771 4365073 : chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
1772 4365073 : : code (code_), dt (dt_), op (op_) {}
1773 : tree_code code;
1774 : vect_def_type dt;
1775 : tree op;
1776 : };
1777 :
1778 : /* Comparator for sorting associatable chains. */
1779 :
1780 : static int
1781 12719169 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
1782 : {
1783 12719169 : auto *op1 = (const chain_op_t *) op1_;
1784 12719169 : auto *op2 = (const chain_op_t *) op2_;
1785 12719169 : if (op1->dt != op2->dt)
1786 1843042 : return (int)op1->dt - (int)op2->dt;
1787 10876127 : return (int)op1->code - (int)op2->code;
1788 : }
1789 :
1790 : /* Linearize the associatable expression chain at START with the
1791 : associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
1792 : filling CHAIN with the result and using WORKLIST as intermediate storage.
1793 : CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
1794 : or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation
1795 : stmts, starting with START. When ALLOW_ALT_CODE is false, do not
1796 : follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf). */
1797 :
1798 : static void
1799 1881433 : vect_slp_linearize_chain (vec_info *vinfo,
1800 : vec<std::pair<tree_code, gimple *> > &worklist,
1801 : vec<chain_op_t> &chain,
1802 : enum tree_code code, gimple *start,
1803 : gimple *&code_stmt, gimple *&alt_code_stmt,
1804 : vec<gimple *> *chain_stmts,
1805 : bool allow_alt_code = true)
1806 : {
1807 : /* For each lane linearize the addition/subtraction (or other
1808 : uniform associatable operation) expression tree. */
1809 1881433 : worklist.safe_push (std::make_pair (code, start));
1810 4365073 : while (!worklist.is_empty ())
1811 : {
1812 2483640 : auto entry = worklist.pop ();
1813 2483640 : gassign *stmt = as_a <gassign *> (entry.second);
1814 2483640 : enum tree_code in_code = entry.first;
1815 4967280 : enum tree_code this_code = gimple_assign_rhs_code (stmt);
1816 : /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE. */
1817 2483640 : if (!code_stmt
1818 2483640 : && gimple_assign_rhs_code (stmt) == code)
1819 1584084 : code_stmt = stmt;
1820 899556 : else if (!alt_code_stmt
1821 899556 : && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
1822 354520 : alt_code_stmt = stmt;
1823 2483640 : if (chain_stmts)
1824 2402497 : chain_stmts->safe_push (stmt);
1825 7450920 : for (unsigned opnum = 1; opnum <= 2; ++opnum)
1826 : {
1827 4967280 : tree op = gimple_op (stmt, opnum);
1828 4967280 : vect_def_type dt;
1829 4967280 : stmt_vec_info def_stmt_info;
1830 4967280 : bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
1831 4967280 : gcc_assert (res);
1832 4967280 : if (dt == vect_internal_def
1833 4967280 : && is_pattern_stmt_p (def_stmt_info))
1834 9255 : op = gimple_get_lhs (def_stmt_info->stmt);
1835 4967280 : gimple *use_stmt;
1836 4967280 : use_operand_p use_p;
1837 4967280 : if (dt == vect_internal_def
1838 4535203 : && single_imm_use (op, &use_p, &use_stmt)
1839 2917828 : && is_gimple_assign (def_stmt_info->stmt)
1840 7691697 : && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
1841 2122525 : || (allow_alt_code
1842 57260 : && code == PLUS_EXPR
1843 36218 : && (gimple_assign_rhs_code (def_stmt_info->stmt)
1844 : == MINUS_EXPR))))
1845 : {
1846 602207 : tree_code op_def_code = this_code;
1847 602207 : if (op_def_code == MINUS_EXPR && opnum == 1)
1848 55385 : op_def_code = PLUS_EXPR;
1849 602207 : if (in_code == MINUS_EXPR)
1850 222 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1851 602207 : worklist.safe_push (std::make_pair (op_def_code,
1852 602207 : def_stmt_info->stmt));
1853 : }
1854 : else
1855 : {
1856 4365073 : tree_code op_def_code = this_code;
1857 4365073 : if (op_def_code == MINUS_EXPR && opnum == 1)
1858 299252 : op_def_code = PLUS_EXPR;
1859 4365073 : if (in_code == MINUS_EXPR)
1860 4278 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1861 4365073 : chain.safe_push (chain_op_t (op_def_code, dt, op));
1862 : }
1863 : }
1864 : }
1865 1881433 : }
1866 :
1867 : /* Distance from the node currently being discovered to the closest upthread
1868 : commutative operation whose operand-zero discovery may still be fixed by
1869 : retrying with swapped operands, or -1U if there is none. */
1870 :
1871 : static unsigned least_upthread_swappable_op_distance = -1U;
1872 :
1873 : static slp_tree
1874 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
1875 : vec<stmt_vec_info> stmts,
1876 : poly_uint64 *max_nunits,
1877 : bool *matches, unsigned *limit, unsigned *tree_size,
1878 : scalar_stmts_to_slp_tree_map_t *bst_map);
1879 :
1880 : static slp_tree
1881 6710806 : vect_build_slp_tree (vec_info *vinfo,
1882 : vec<stmt_vec_info> stmts,
1883 : poly_uint64 *max_nunits,
1884 : bool *matches, unsigned *limit, unsigned *tree_size,
1885 : scalar_stmts_to_slp_tree_map_t *bst_map)
1886 : {
1887 6710806 : unsigned int group_size = stmts.length ();
1888 6710806 : if (slp_tree *leader = bst_map->get (stmts))
1889 : {
1890 501157 : if (dump_enabled_p ())
1891 17368 : dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
1892 17368 : !(*leader)->failed ? "" : "failed ",
1893 : (void *) *leader);
1894 501157 : if (!(*leader)->failed)
1895 : {
1896 451502 : SLP_TREE_REF_COUNT (*leader)++;
1897 451502 : vect_update_max_nunits (max_nunits, (*leader)->max_nunits);
1898 451502 : stmts.release ();
1899 451502 : return *leader;
1900 : }
1901 49655 : memcpy (matches, (*leader)->failed, sizeof (bool) * group_size);
1902 49655 : return NULL;
1903 : }
1904 :
1905 : /* Single-lane SLP doesn't have the chance of run-away, do not account
1906 : it to the limit. */
1907 6209649 : if (stmts.length () > 1)
1908 : {
1909 3514883 : if (*limit == 0)
1910 : {
1911 1293 : if (dump_enabled_p ())
1912 15 : dump_printf_loc (MSG_NOTE, vect_location,
1913 : "SLP discovery limit exceeded\n");
1914 1293 : memset (matches, 0, sizeof (bool) * group_size);
1915 1293 : return NULL;
1916 : }
1917 3513590 : --*limit;
1918 : }
1919 :
1920 : /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
1921 : so we can pick up backedge destinations during discovery. */
1922 6208356 : slp_tree res = new _slp_tree;
1923 6208356 : SLP_TREE_DEF_TYPE (res) = vect_internal_def;
1924 6208356 : SLP_TREE_SCALAR_STMTS (res) = stmts;
1925 6208356 : bst_map->put (stmts.copy (), res);
1926 :
1927 6208356 : if (dump_enabled_p ())
1928 150120 : dump_printf_loc (MSG_NOTE, vect_location,
1929 : "starting SLP discovery for node %p\n", (void *) res);
1930 :
1931 6208356 : poly_uint64 this_max_nunits = 1;
1932 6208356 : slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
1933 : &this_max_nunits,
1934 : matches, limit, tree_size, bst_map);
1935 6208356 : if (!res_)
1936 : {
1937 2251103 : if (dump_enabled_p ())
1938 7822 : dump_printf_loc (MSG_NOTE, vect_location,
1939 : "SLP discovery for node %p failed\n", (void *) res);
1940 : /* Mark the node invalid so we can detect those when still in use
1941 : as backedge destinations. */
1942 2251103 : SLP_TREE_SCALAR_STMTS (res) = vNULL;
1943 2251103 : SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
1944 2251103 : res->failed = XNEWVEC (bool, group_size);
1945 2251103 : if (flag_checking)
1946 : {
1947 : unsigned i;
1948 4072625 : for (i = 0; i < group_size; ++i)
1949 4072625 : if (!matches[i])
1950 : break;
1951 2251103 : gcc_assert (i < group_size);
1952 : }
1953 2251103 : memcpy (res->failed, matches, sizeof (bool) * group_size);
1954 : }
1955 : else
1956 : {
1957 3957253 : if (dump_enabled_p ())
1958 142298 : dump_printf_loc (MSG_NOTE, vect_location,
1959 : "SLP discovery for node %p succeeded\n",
1960 : (void *) res);
1961 3957253 : gcc_assert (res_ == res);
1962 3957253 : res->max_nunits = this_max_nunits;
1963 3957253 : vect_update_max_nunits (max_nunits, this_max_nunits);
1964 : /* Keep a reference for the bst_map use. */
1965 3957253 : SLP_TREE_REF_COUNT (res)++;
1966 : }
1967 : return res_;
1968 : }
1969 :
1970 : /* Helper for building an associated SLP node chain. */
1971 :
1972 : static void
1973 158 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
1974 : slp_tree op0, slp_tree op1,
1975 : stmt_vec_info oper1, stmt_vec_info oper2,
1976 : vec<std::pair<unsigned, unsigned> > lperm)
1977 : {
1978 158 : unsigned group_size = SLP_TREE_LANES (op1);
1979 :
1980 158 : slp_tree child1 = new _slp_tree;
1981 158 : SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
1982 158 : SLP_TREE_VECTYPE (child1) = vectype;
1983 158 : SLP_TREE_LANES (child1) = group_size;
1984 158 : SLP_TREE_CHILDREN (child1).create (2);
1985 158 : SLP_TREE_CHILDREN (child1).quick_push (op0);
1986 158 : SLP_TREE_CHILDREN (child1).quick_push (op1);
1987 158 : SLP_TREE_REPRESENTATIVE (child1) = oper1;
1988 :
1989 158 : slp_tree child2 = new _slp_tree;
1990 158 : SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
1991 158 : SLP_TREE_VECTYPE (child2) = vectype;
1992 158 : SLP_TREE_LANES (child2) = group_size;
1993 158 : SLP_TREE_CHILDREN (child2).create (2);
1994 158 : SLP_TREE_CHILDREN (child2).quick_push (op0);
1995 158 : SLP_TREE_REF_COUNT (op0)++;
1996 158 : SLP_TREE_CHILDREN (child2).quick_push (op1);
1997 158 : SLP_TREE_REF_COUNT (op1)++;
1998 158 : SLP_TREE_REPRESENTATIVE (child2) = oper2;
1999 :
2000 158 : SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
2001 158 : SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
2002 158 : SLP_TREE_VECTYPE (perm) = vectype;
2003 158 : SLP_TREE_LANES (perm) = group_size;
2004 : /* ??? We should set this NULL but that's not expected. */
2005 158 : SLP_TREE_REPRESENTATIVE (perm) = oper1;
2006 158 : SLP_TREE_LANE_PERMUTATION (perm) = lperm;
2007 158 : SLP_TREE_CHILDREN (perm).quick_push (child1);
2008 158 : SLP_TREE_CHILDREN (perm).quick_push (child2);
2009 158 : }
2010 :
2011 : /* Recursively build an SLP tree starting from NODE.
2012 : Fail (and return a value not equal to zero) if def-stmts are not
2013 : isomorphic, require data permutation or are of unsupported types of
2014 : operation. Otherwise, return 0.
2015 : The value returned is the depth in the SLP tree where a mismatch
2016 : was found. */
2017 :
2018 : static slp_tree
2019 6208356 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
2020 : vec<stmt_vec_info> stmts,
2021 : poly_uint64 *max_nunits,
2022 : bool *matches, unsigned *limit, unsigned *tree_size,
2023 : scalar_stmts_to_slp_tree_map_t *bst_map)
2024 : {
2025 6208356 : unsigned int group_size = stmts.length ();
2026 6208356 : unsigned nops, i, this_tree_size = 0;
2027 6208356 : poly_uint64 this_max_nunits = *max_nunits;
2028 :
2029 6208356 : matches[0] = false;
2030 :
2031 6208356 : stmt_vec_info stmt_info = stmts[0];
2032 6208356 : if (!is_a<gcall *> (stmt_info->stmt)
2033 : && !is_a<gassign *> (stmt_info->stmt)
2034 : && !is_a<gphi *> (stmt_info->stmt))
2035 : return NULL;
2036 :
2037 6208258 : nops = gimple_num_args (stmt_info->stmt);
2038 6208258 : if (const int *map = vect_get_operand_map (stmt_info))
2039 37037 : nops = map[0];
2040 :
2041 : /* If the SLP node is a PHI (induction or reduction), terminate
2042 : the recursion. */
2043 6208258 : bool *skip_args = XALLOCAVEC (bool, nops);
2044 6208258 : memset (skip_args, 0, sizeof (bool) * nops);
2045 6208258 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2046 2893135 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
2047 : {
2048 315498 : tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
2049 315498 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
2050 : group_size);
2051 315498 : if (!vect_record_max_nunits (vinfo, stmt_info, group_size, vectype,
2052 : max_nunits))
2053 : return NULL;
2054 :
2055 311470 : vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
2056 311470 : if (def_type == vect_induction_def)
2057 : {
2058 : /* Induction PHIs are not cycles but walk the initial
2059 : value. Only for inner loops through, for outer loops
2060 : we need to pick up the value from the actual PHIs
2061 : to more easily support peeling and epilogue vectorization. */
2062 203216 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2063 203216 : if (!nested_in_vect_loop_p (loop, stmt_info))
2064 202357 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2065 : else
2066 : loop = loop->inner;
2067 203216 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2068 : }
2069 108254 : else if (def_type == vect_reduction_def
2070 : || def_type == vect_double_reduction_def
2071 : || def_type == vect_nested_cycle
2072 108254 : || def_type == vect_first_order_recurrence)
2073 : {
2074 : /* Else def types have to match. */
2075 : stmt_vec_info other_info;
2076 : bool all_same = true;
2077 244712 : FOR_EACH_VEC_ELT (stmts, i, other_info)
2078 : {
2079 137784 : if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
2080 1993439 : return NULL;
2081 137778 : if (other_info != stmt_info)
2082 26228 : all_same = false;
2083 : }
2084 106928 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2085 : /* Reduction initial values are not explicitly represented. */
2086 106928 : if (def_type != vect_first_order_recurrence
2087 106928 : && gimple_bb (stmt_info->stmt) == loop->header)
2088 103681 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2089 : /* Reduction chain backedge defs are filled manually.
2090 : ??? Need a better way to identify a SLP reduction chain PHI.
2091 : Or a better overall way to SLP match those. */
2092 106928 : if (stmts.length () > 1
2093 106928 : && all_same && def_type == vect_reduction_def)
2094 2364 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2095 : }
2096 1320 : else if (def_type != vect_internal_def)
2097 : return NULL;
2098 : }
2099 :
2100 :
2101 6204224 : bool two_operators = false;
2102 6204224 : unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
2103 6204224 : tree vectype = NULL_TREE;
2104 6204224 : if (!vect_build_slp_tree_1 (vinfo, swap, stmts,
2105 : &this_max_nunits, matches, &two_operators,
2106 : &vectype))
2107 : return NULL;
2108 :
2109 : /* If the SLP node is a load, terminate the recursion unless masked. */
2110 4453460 : if (STMT_VINFO_DATA_REF (stmt_info)
2111 2097900 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
2112 : {
2113 945289 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2114 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
2115 : else
2116 : {
2117 925739 : *max_nunits = this_max_nunits;
2118 925739 : (*tree_size)++;
2119 925739 : node = vect_create_new_slp_node (node, stmts, 0);
2120 925739 : SLP_TREE_VECTYPE (node) = vectype;
2121 : /* And compute the load permutation. Whether it is actually
2122 : a permutation depends on the unrolling factor which is
2123 : decided later. */
2124 925739 : vec<unsigned> load_permutation;
2125 925739 : int j;
2126 925739 : stmt_vec_info load_info;
2127 925739 : load_permutation.create (group_size);
2128 925739 : stmt_vec_info first_stmt_info
2129 925739 : = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2130 925739 : ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
2131 925739 : bool any_permute = false;
2132 2243029 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
2133 : {
2134 1317290 : int load_place;
2135 1317290 : if (! load_info)
2136 : {
2137 40839 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2138 : load_place = j;
2139 : else
2140 : load_place = 0;
2141 : }
2142 1276451 : else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2143 749780 : load_place = vect_get_place_in_interleaving_chain
2144 749780 : (load_info, first_stmt_info);
2145 : else
2146 : /* Recognize the splat case as { 0, 0, ... } but make
2147 : sure to use the appropriate refs for collections
2148 : of invariant refs. */
2149 526671 : load_place = (load_info == stmt_info) ? 0 : j;
2150 790860 : gcc_assert (load_place != -1);
2151 1317290 : any_permute |= load_place != j;
2152 1317290 : load_permutation.quick_push (load_place);
2153 : }
2154 :
2155 925739 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
2156 : {
2157 3500 : gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
2158 3500 : bool has_gaps = false;
2159 3500 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2160 189 : for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
2161 846 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2162 657 : if (DR_GROUP_GAP (si) != 1)
2163 80 : has_gaps = true;
2164 : /* We cannot handle permuted masked loads directly, see
2165 : PR114375. We cannot handle strided masked loads or masked
2166 : loads with gaps unless the mask is uniform. */
2167 3500 : if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
2168 189 : && (DR_GROUP_GAP (first_stmt_info) != 0
2169 129 : || (has_gaps
2170 35 : && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
2171 6920 : || STMT_VINFO_STRIDED_P (stmt_info))
2172 : {
2173 93 : load_permutation.release ();
2174 93 : matches[0] = false;
2175 922371 : return NULL;
2176 : }
2177 :
2178 : /* For permuted masked loads do an unpermuted masked load of
2179 : the whole group followed by a SLP permute node. */
2180 3407 : if (any_permute
2181 3407 : || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
2182 83 : && DR_GROUP_SIZE (first_stmt_info) != group_size))
2183 : {
2184 : /* Discover the whole unpermuted load. */
2185 39 : vec<stmt_vec_info> stmts2;
2186 39 : unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2187 68 : ? DR_GROUP_SIZE (first_stmt_info) : 1;
2188 39 : stmts2.create (dr_group_size);
2189 39 : stmts2.quick_grow_cleared (dr_group_size);
2190 39 : unsigned i = 0;
2191 39 : for (stmt_vec_info si = first_stmt_info;
2192 464 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2193 : {
2194 425 : if (si != first_stmt_info)
2195 1586 : for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
2196 1200 : stmts2[i++] = NULL;
2197 425 : stmts2[i++] = si;
2198 : }
2199 39 : bool *matches2 = XALLOCAVEC (bool, dr_group_size);
2200 39 : slp_tree unperm_load
2201 39 : = vect_build_slp_tree (vinfo, stmts2,
2202 : &this_max_nunits, matches2, limit,
2203 39 : &this_tree_size, bst_map);
2204 : /* When we are able to do the full masked load emit that
2205 : followed by 'node' being the desired final permutation. */
2206 39 : if (unperm_load)
2207 : {
2208 16 : gcc_assert
2209 : (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
2210 16 : lane_permutation_t lperm;
2211 16 : lperm.create (group_size);
2212 56 : for (unsigned j = 0; j < load_permutation.length (); ++j)
2213 40 : lperm.quick_push
2214 40 : (std::make_pair (0, load_permutation[j]));
2215 16 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2216 16 : SLP_TREE_CHILDREN (node).safe_push (unperm_load);
2217 16 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2218 16 : load_permutation.release ();
2219 16 : return node;
2220 : }
2221 23 : stmts2.release ();
2222 23 : load_permutation.release ();
2223 23 : matches[0] = false;
2224 23 : return NULL;
2225 : }
2226 3368 : load_permutation.release ();
2227 : }
2228 : else
2229 : {
2230 922239 : if (!any_permute
2231 793659 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2232 1215559 : && group_size == DR_GROUP_SIZE (first_stmt_info))
2233 128703 : load_permutation.release ();
2234 922239 : SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
2235 922239 : return node;
2236 : }
2237 : }
2238 : }
2239 3508171 : else if (gimple_assign_single_p (stmt_info->stmt)
2240 2312060 : && !gimple_vuse (stmt_info->stmt)
2241 3517034 : && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
2242 : {
2243 : /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
2244 : the same SSA name vector of a compatible type to vectype. */
2245 3386 : vec<std::pair<unsigned, unsigned> > lperm = vNULL;
2246 3386 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
2247 3386 : stmt_vec_info estmt_info;
2248 11982 : FOR_EACH_VEC_ELT (stmts, i, estmt_info)
2249 : {
2250 8743 : gassign *estmt = as_a <gassign *> (estmt_info->stmt);
2251 8743 : tree bfref = gimple_assign_rhs1 (estmt);
2252 8743 : HOST_WIDE_INT lane;
2253 8743 : if (!known_eq (bit_field_size (bfref),
2254 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
2255 17339 : || !constant_multiple_p (bit_field_offset (bfref),
2256 8596 : bit_field_size (bfref), &lane))
2257 : {
2258 147 : lperm.release ();
2259 147 : matches[0] = false;
2260 147 : return NULL;
2261 : }
2262 8596 : lperm.safe_push (std::make_pair (0, (unsigned)lane));
2263 : }
2264 3239 : slp_tree vnode = vect_create_new_slp_node (vNULL);
2265 3239 : if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
2266 : /* ??? We record vectype here but we hide eventually necessary
2267 : punning and instead rely on code generation to materialize
2268 : VIEW_CONVERT_EXPRs as necessary. We instead should make
2269 : this explicit somehow. */
2270 1445 : SLP_TREE_VECTYPE (vnode) = vectype;
2271 : else
2272 : {
2273 : /* For different size but compatible elements we can still
2274 : use VEC_PERM_EXPR without punning. */
2275 1794 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
2276 : && types_compatible_p (TREE_TYPE (vectype),
2277 : TREE_TYPE (TREE_TYPE (vec))));
2278 1794 : SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
2279 : }
2280 3239 : auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
2281 3239 : unsigned HOST_WIDE_INT const_nunits;
2282 3239 : if (nunits.is_constant (&const_nunits))
2283 3239 : SLP_TREE_LANES (vnode) = const_nunits;
2284 3239 : SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
2285 : /* We are always building a permutation node even if it is an identity
2286 : permute to shield the rest of the vectorizer from the odd node
2287 : representing an actual vector without any scalar ops.
2288 : ??? We could hide it completely with making the permute node
2289 : external? */
2290 3239 : node = vect_create_new_slp_node (node, stmts, 1);
2291 3239 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2292 3239 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2293 3239 : SLP_TREE_VECTYPE (node) = vectype;
2294 3239 : SLP_TREE_CHILDREN (node).quick_push (vnode);
2295 3239 : return node;
2296 : }
2297 : /* When discovery reaches an associatable operation see whether we can
2298 : improve that to match up lanes in a way superior to the operand
2299 : swapping code which at most looks at two defs.
2300 : ??? For BB vectorization we cannot do the brute-force search
2301 : for matching as we can succeed by means of builds from scalars
2302 : and have no good way to "cost" one build against another. */
2303 3504785 : else if (is_a <loop_vec_info> (vinfo)
2304 : /* Do not bother for single-lane SLP. */
2305 2056749 : && group_size > 1
2306 : /* ??? We don't handle !vect_internal_def defs below. */
2307 113605 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
2308 : /* ??? Do not associate a reduction, this will wreck REDUC_IDX
2309 : mapping as long as that exists on the stmt_info level. */
2310 88062 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1
2311 79513 : && is_gimple_assign (stmt_info->stmt)
2312 79194 : && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
2313 52334 : || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
2314 3533595 : && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
2315 16555 : || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
2316 14021 : && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
2317 : {
2318 : /* See if we have a chain of (mixed) adds or subtracts or other
2319 : associatable ops. */
2320 21767 : enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
2321 21767 : if (code == MINUS_EXPR)
2322 848 : code = PLUS_EXPR;
2323 21767 : stmt_vec_info other_op_stmt_info = NULL;
2324 21767 : stmt_vec_info op_stmt_info = NULL;
2325 21767 : unsigned chain_len = 0;
2326 21767 : auto_vec<chain_op_t> chain;
2327 21767 : auto_vec<std::pair<tree_code, gimple *> > worklist;
2328 21767 : auto_vec<vec<chain_op_t> > chains (group_size);
2329 21767 : auto_vec<slp_tree, 4> children;
2330 21767 : bool hard_fail = true;
2331 22794 : for (unsigned lane = 0; lane < group_size; ++lane)
2332 : {
2333 22458 : if (!stmts[lane])
2334 : {
2335 : /* ??? Below we require lane zero is present. */
2336 0 : if (lane == 0)
2337 : {
2338 : hard_fail = false;
2339 21431 : break;
2340 : }
2341 0 : chains.quick_push (vNULL);
2342 0 : continue;
2343 : }
2344 : /* For each lane linearize the addition/subtraction (or other
2345 : uniform associatable operation) expression tree. */
2346 22458 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
2347 22458 : vect_slp_linearize_chain (vinfo, worklist, chain, code,
2348 22458 : stmts[lane]->stmt, op_stmt, other_op_stmt,
2349 : NULL);
2350 22458 : if (!op_stmt_info && op_stmt)
2351 21116 : op_stmt_info = vinfo->lookup_stmt (op_stmt);
2352 22458 : if (!other_op_stmt_info && other_op_stmt)
2353 884 : other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
2354 22458 : if (chain.length () == 2)
2355 : {
2356 : /* In a chain of just two elements resort to the regular
2357 : operand swapping scheme. Likewise if we run into a
2358 : length mismatch process regularly as well as we did not
2359 : process the other lanes we cannot report a good hint what
2360 : lanes to try swapping in the parent. */
2361 : hard_fail = false;
2362 : break;
2363 : }
2364 1030 : else if (chain_len == 0)
2365 376 : chain_len = chain.length ();
2366 1308 : else if (chain.length () != chain_len)
2367 : {
2368 : /* ??? Here we could slip in magic to compensate with
2369 : neutral operands. */
2370 3 : matches[lane] = false;
2371 3 : if (lane != group_size - 1)
2372 3 : matches[0] = false;
2373 : break;
2374 : }
2375 1027 : chains.quick_push (chain.copy ());
2376 1027 : chain.truncate (0);
2377 : }
2378 43534 : if (chains.length () == group_size)
2379 : {
2380 : /* We cannot yet use SLP_TREE_CODE to communicate the operation. */
2381 336 : if (!op_stmt_info)
2382 : {
2383 3 : hard_fail = false;
2384 3 : goto out;
2385 : }
2386 : /* Now we have a set of chains with the same length. */
2387 : /* 1. pre-sort according to def_type and operation. */
2388 1248 : for (unsigned lane = 0; lane < group_size; ++lane)
2389 1830 : chains[lane].stablesort (dt_sort_cmp, vinfo);
2390 333 : if (dump_enabled_p ())
2391 : {
2392 157 : dump_printf_loc (MSG_NOTE, vect_location,
2393 : "pre-sorted chains of %s\n",
2394 : get_tree_code_name (code));
2395 685 : for (unsigned lane = 0; lane < group_size; ++lane)
2396 : {
2397 528 : if (!stmts[lane])
2398 0 : dump_printf (MSG_NOTE, "--");
2399 : else
2400 2422 : for (unsigned opnum = 0; opnum < chain_len; ++opnum)
2401 3788 : dump_printf (MSG_NOTE, "%s %T ",
2402 1894 : get_tree_code_name (chains[lane][opnum].code),
2403 1894 : chains[lane][opnum].op);
2404 528 : dump_printf (MSG_NOTE, "\n");
2405 : }
2406 : }
2407 : /* 2. try to build children nodes, associating as necessary. */
2408 : /* 2a. prepare and perform early checks to avoid eating into
2409 : discovery limit unnecessarily. */
2410 333 : vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
2411 1407 : for (unsigned n = 0; n < chain_len; ++n)
2412 : {
2413 1074 : vect_def_type dt = chains[0][n].dt;
2414 1074 : unsigned lane;
2415 4177 : for (lane = 0; lane < group_size; ++lane)
2416 6206 : if (stmts[lane] && chains[lane][n].dt != dt)
2417 : {
2418 0 : if (dt == vect_constant_def
2419 0 : && chains[lane][n].dt == vect_external_def)
2420 : dt = vect_external_def;
2421 0 : else if (dt == vect_external_def
2422 0 : && chains[lane][n].dt == vect_constant_def)
2423 : ;
2424 : else
2425 : break;
2426 : }
2427 1074 : if (lane != group_size)
2428 : {
2429 0 : if (dump_enabled_p ())
2430 0 : dump_printf_loc (MSG_NOTE, vect_location,
2431 : "giving up on chain due to mismatched "
2432 : "def types\n");
2433 0 : matches[lane] = false;
2434 0 : if (lane != group_size - 1)
2435 0 : matches[0] = false;
2436 0 : goto out;
2437 : }
2438 1074 : dts[n] = dt;
2439 1074 : if (dt == vect_constant_def
2440 1074 : || dt == vect_external_def)
2441 : {
2442 : /* Check whether we can build the invariant. If we can't
2443 : we never will be able to. */
2444 93 : tree type = TREE_TYPE (chains[0][n].op);
2445 1074 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
2446 : && (TREE_CODE (type) == BOOLEAN_TYPE
2447 : || !can_duplicate_and_interleave_p (vinfo, group_size,
2448 : type)))
2449 : {
2450 : matches[0] = false;
2451 : goto out;
2452 : }
2453 : }
2454 981 : else if (dt != vect_internal_def)
2455 : {
2456 : /* Not sure, we might need sth special.
2457 : gcc.dg/vect/pr96854.c,
2458 : gfortran.dg/vect/fast-math-pr37021.f90
2459 : and gfortran.dg/vect/pr61171.f trigger. */
2460 : /* Soft-fail for now. */
2461 0 : hard_fail = false;
2462 0 : goto out;
2463 : }
2464 : }
2465 : /* 2b. do the actual build. */
2466 1353 : for (unsigned n = 0; n < chain_len; ++n)
2467 : {
2468 1039 : vect_def_type dt = dts[n];
2469 1039 : unsigned lane;
2470 1039 : if (dt == vect_constant_def
2471 1039 : || dt == vect_external_def)
2472 : {
2473 93 : vec<tree> ops;
2474 93 : ops.create (group_size);
2475 461 : for (lane = 0; lane < group_size; ++lane)
2476 275 : if (stmts[lane])
2477 275 : ops.quick_push (chains[lane][n].op);
2478 : else
2479 0 : ops.quick_push (NULL_TREE);
2480 93 : slp_tree child = vect_create_new_slp_node (ops);
2481 93 : SLP_TREE_DEF_TYPE (child) = dt;
2482 93 : children.safe_push (child);
2483 : }
2484 : else
2485 : {
2486 946 : vec<stmt_vec_info> op_stmts;
2487 946 : op_stmts.create (group_size);
2488 946 : slp_tree child = NULL;
2489 : /* Brute-force our way. We have to consider a lane
2490 : failing after fixing an earlier fail up in the
2491 : SLP discovery recursion. So track the current
2492 : permute per lane. */
2493 946 : unsigned *perms = XALLOCAVEC (unsigned, group_size);
2494 946 : memset (perms, 0, sizeof (unsigned) * group_size);
2495 1040 : do
2496 : {
2497 1040 : op_stmts.truncate (0);
2498 5092 : for (lane = 0; lane < group_size; ++lane)
2499 3012 : if (stmts[lane])
2500 3012 : op_stmts.quick_push
2501 3012 : (vinfo->lookup_def (chains[lane][n].op));
2502 : else
2503 0 : op_stmts.quick_push (NULL);
2504 1040 : child = vect_build_slp_tree (vinfo, op_stmts,
2505 : &this_max_nunits,
2506 : matches, limit,
2507 : &this_tree_size, bst_map);
2508 : /* ??? We're likely getting too many fatal mismatches
2509 : here so maybe we want to ignore them (but then we
2510 : have no idea which lanes fatally mismatched). */
2511 1040 : if (child || !matches[0])
2512 : break;
2513 : /* Swap another lane we have not yet matched up into
2514 : lanes that did not match. If we run out of
2515 : permute possibilities for a lane terminate the
2516 : search. */
2517 287 : bool term = false;
2518 287 : for (lane = 1; lane < group_size; ++lane)
2519 193 : if (!matches[lane])
2520 : {
2521 165 : if (n + perms[lane] + 1 == chain_len)
2522 : {
2523 : term = true;
2524 : break;
2525 : }
2526 146 : if (dump_enabled_p ())
2527 113 : dump_printf_loc (MSG_NOTE, vect_location,
2528 : "swapping operand %d and %d "
2529 : "of lane %d\n",
2530 : n, n + perms[lane] + 1, lane);
2531 292 : std::swap (chains[lane][n],
2532 146 : chains[lane][n + perms[lane] + 1]);
2533 146 : perms[lane]++;
2534 : }
2535 113 : if (term)
2536 : break;
2537 : }
2538 : while (1);
2539 946 : if (!child)
2540 : {
2541 19 : if (dump_enabled_p ())
2542 18 : dump_printf_loc (MSG_NOTE, vect_location,
2543 : "failed to match up op %d\n", n);
2544 19 : op_stmts.release ();
2545 19 : if (lane != group_size - 1)
2546 9 : matches[0] = false;
2547 : else
2548 10 : matches[lane] = false;
2549 19 : goto out;
2550 : }
2551 927 : if (dump_enabled_p ())
2552 : {
2553 421 : dump_printf_loc (MSG_NOTE, vect_location,
2554 : "matched up op %d to\n", n);
2555 421 : vect_print_slp_tree (MSG_NOTE, vect_location, child);
2556 : }
2557 927 : children.safe_push (child);
2558 : }
2559 : }
2560 : /* 3. build SLP nodes to combine the chain. */
2561 1156 : for (unsigned lane = 0; lane < group_size; ++lane)
2562 1696 : if (stmts[lane] && chains[lane][0].code != code)
2563 : {
2564 : /* See if there's any alternate all-PLUS entry. */
2565 : unsigned n;
2566 6 : for (n = 1; n < chain_len; ++n)
2567 : {
2568 30 : for (lane = 0; lane < group_size; ++lane)
2569 48 : if (stmts[lane] && chains[lane][n].code != code)
2570 : break;
2571 6 : if (lane == group_size)
2572 : break;
2573 : }
2574 6 : if (n != chain_len)
2575 : {
2576 : /* Swap that in at first position. */
2577 6 : std::swap (children[0], children[n]);
2578 30 : for (lane = 0; lane < group_size; ++lane)
2579 24 : if (stmts[lane])
2580 24 : std::swap (chains[lane][0], chains[lane][n]);
2581 : }
2582 : else
2583 : {
2584 : /* ??? When this triggers and we end up with two
2585 : vect_constant/external_def up-front things break (ICE)
2586 : spectacularly finding an insertion place for the
2587 : all-constant op. We should have a fully
2588 : vect_internal_def operand though(?) so we can swap
2589 : that into first place and then prepend the all-zero
2590 : constant. */
2591 0 : if (dump_enabled_p ())
2592 0 : dump_printf_loc (MSG_NOTE, vect_location,
2593 : "inserting constant zero to compensate "
2594 : "for (partially) negated first "
2595 : "operand\n");
2596 0 : chain_len++;
2597 0 : for (lane = 0; lane < group_size; ++lane)
2598 0 : if (stmts[lane])
2599 0 : chains[lane].safe_insert
2600 0 : (0, chain_op_t (code, vect_constant_def, NULL_TREE));
2601 0 : vec<tree> zero_ops;
2602 0 : zero_ops.create (group_size);
2603 0 : zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
2604 0 : for (lane = 1; lane < group_size; ++lane)
2605 0 : if (stmts[lane])
2606 0 : zero_ops.quick_push (zero_ops[0]);
2607 : else
2608 0 : zero_ops.quick_push (NULL_TREE);
2609 0 : slp_tree zero = vect_create_new_slp_node (zero_ops);
2610 0 : SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
2611 0 : children.safe_insert (0, zero);
2612 : }
2613 : break;
2614 : }
2615 1015 : for (unsigned i = 1; i < children.length (); ++i)
2616 : {
2617 701 : slp_tree op0 = children[i - 1];
2618 701 : slp_tree op1 = children[i];
2619 701 : bool this_two_op = false;
2620 2569 : for (unsigned lane = 0; lane < group_size; ++lane)
2621 4052 : if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
2622 : {
2623 : this_two_op = true;
2624 : break;
2625 : }
2626 701 : slp_tree child;
2627 701 : if (i == children.length () - 1)
2628 314 : child = vect_create_new_slp_node (node, stmts, 2);
2629 : else
2630 387 : child = vect_create_new_slp_node (2, ERROR_MARK);
2631 701 : if (this_two_op)
2632 : {
2633 158 : vec<std::pair<unsigned, unsigned> > lperm;
2634 158 : lperm.create (group_size);
2635 570 : for (unsigned lane = 0; lane < group_size; ++lane)
2636 824 : lperm.quick_push (std::make_pair
2637 412 : (chains[lane][i].code != chains[0][i].code, lane));
2638 316 : vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
2639 158 : (chains[0][i].code == code
2640 : ? op_stmt_info
2641 : : other_op_stmt_info),
2642 158 : (chains[0][i].code == code
2643 : ? other_op_stmt_info
2644 : : op_stmt_info),
2645 : lperm);
2646 : }
2647 : else
2648 : {
2649 543 : SLP_TREE_DEF_TYPE (child) = vect_internal_def;
2650 543 : SLP_TREE_VECTYPE (child) = vectype;
2651 543 : SLP_TREE_LANES (child) = group_size;
2652 543 : SLP_TREE_CHILDREN (child).quick_push (op0);
2653 543 : SLP_TREE_CHILDREN (child).quick_push (op1);
2654 543 : SLP_TREE_REPRESENTATIVE (child)
2655 1086 : = (chains[0][i].code == code
2656 543 : ? op_stmt_info : other_op_stmt_info);
2657 : }
2658 701 : children[i] = child;
2659 : }
2660 314 : *tree_size += this_tree_size + 1;
2661 314 : *max_nunits = this_max_nunits;
2662 1516 : while (!chains.is_empty ())
2663 866 : chains.pop ().release ();
2664 : return node;
2665 : }
2666 21431 : out:
2667 21453 : if (dump_enabled_p ())
2668 2817 : dump_printf_loc (MSG_NOTE, vect_location,
2669 : "failed to line up SLP graph by re-associating "
2670 : "operations in lanes%s\n",
2671 : !hard_fail ? " trying regular discovery" : "");
2672 21458 : while (!children.is_empty ())
2673 5 : vect_free_slp_tree (children.pop ());
2674 21614 : while (!chains.is_empty ())
2675 161 : chains.pop ().release ();
2676 : /* Hard-fail, otherwise we might run into quadratic processing of the
2677 : chains starting one stmt into the chain again. */
2678 21453 : if (hard_fail)
2679 : return NULL;
2680 : /* Fall thru to normal processing. */
2681 21767 : }
2682 :
2683 : /* Get at the operands, verifying they are compatible. */
2684 3527367 : vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
2685 3527367 : slp_oprnd_info oprnd_info;
2686 16591974 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
2687 : {
2688 26131706 : int res = vect_get_and_check_slp_defs (vinfo, vectype,
2689 13065853 : swap[i], skip_args,
2690 : stmts, i, &oprnds_info);
2691 13065853 : if (res != 0)
2692 599832 : matches[(res == -1) ? 0 : i] = false;
2693 13065853 : if (!matches[0])
2694 : break;
2695 : }
2696 16252034 : for (i = 0; i < group_size; ++i)
2697 12963210 : if (!matches[i])
2698 : {
2699 238543 : vect_free_oprnd_info (oprnds_info);
2700 238543 : return NULL;
2701 : }
2702 9866472 : swap = NULL;
2703 :
2704 9866472 : bool has_two_operators_perm = false;
2705 19732944 : auto_vec<unsigned> two_op_perm_indices[2];
2706 3288824 : vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
2707 :
2708 3304901 : if (two_operators && oprnds_info.length () == 2 && group_size > 2)
2709 : {
2710 4012 : unsigned idx = 0;
2711 4012 : hash_map<gimple *, unsigned> seen;
2712 4012 : vec<slp_oprnd_info> new_oprnds_info
2713 4012 : = vect_create_oprnd_info (1, group_size);
2714 4012 : bool success = true;
2715 :
2716 4012 : enum tree_code code = ERROR_MARK;
2717 4012 : if (oprnds_info[0]->def_stmts[0]
2718 4012 : && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
2719 3951 : code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
2720 4012 : basic_block bb = nullptr;
2721 :
2722 7708 : for (unsigned j = 0; j < group_size; ++j)
2723 : {
2724 18015 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2725 : {
2726 14319 : stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
2727 14319 : if (!stmt_info
2728 14153 : || !is_a<gassign *> (stmt_info->stmt)
2729 14150 : || gimple_assign_rhs_code (stmt_info->stmt) != code
2730 25109 : || skip_args[i])
2731 : {
2732 : success = false;
2733 3533 : break;
2734 : }
2735 : /* Avoid mixing lanes with defs in different basic-blocks. */
2736 10790 : if (!bb)
2737 4147 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
2738 8577 : else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
2739 : {
2740 : success = false;
2741 : break;
2742 : }
2743 :
2744 10786 : bool exists;
2745 10786 : unsigned &stmt_idx
2746 10786 : = seen.get_or_insert (stmt_info->stmt, &exists);
2747 :
2748 10786 : if (!exists)
2749 : {
2750 9405 : new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
2751 9405 : new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
2752 9405 : stmt_idx = idx;
2753 9405 : idx++;
2754 : }
2755 :
2756 10786 : two_op_perm_indices[i].safe_push (stmt_idx);
2757 : }
2758 :
2759 7229 : if (!success)
2760 : break;
2761 : }
2762 :
2763 4012 : if (success && idx == group_size)
2764 : {
2765 98 : if (dump_enabled_p ())
2766 : {
2767 0 : dump_printf_loc (MSG_NOTE, vect_location,
2768 : "Replace two_operators operands:\n");
2769 :
2770 0 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2771 : {
2772 0 : dump_printf_loc (MSG_NOTE, vect_location,
2773 : "Operand %u:\n", i);
2774 0 : for (unsigned j = 0; j < group_size; j++)
2775 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2776 0 : j, oprnd_info->def_stmts[j]->stmt);
2777 : }
2778 :
2779 0 : dump_printf_loc (MSG_NOTE, vect_location,
2780 : "With a single operand:\n");
2781 0 : for (unsigned j = 0; j < group_size; j++)
2782 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2783 0 : j, new_oprnds_info[0]->def_stmts[j]->stmt);
2784 : }
2785 :
2786 98 : two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
2787 98 : two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
2788 :
2789 98 : new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
2790 98 : new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
2791 98 : new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
2792 98 : new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
2793 98 : new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
2794 :
2795 98 : vect_free_oprnd_info (oprnds_info);
2796 98 : oprnds_info = new_oprnds_info;
2797 98 : nops = 1;
2798 98 : has_two_operators_perm = true;
2799 : }
2800 : else
2801 3914 : vect_free_oprnd_info (new_oprnds_info);
2802 4012 : }
2803 :
2804 6577648 : auto_vec<slp_tree, 4> children;
2805 :
2806 3288824 : stmt_info = stmts[0];
2807 :
2808 3288824 : int reduc_idx = -1;
2809 3288824 : int gs_scale = 0;
2810 3288824 : tree gs_base = NULL_TREE;
2811 :
2812 : /* Create SLP_TREE nodes for the definition node/s. */
2813 8456258 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2814 : {
2815 5284977 : slp_tree child = nullptr;
2816 5284977 : unsigned int j;
2817 5284977 : unsigned old_swap_distance;
2818 5284977 : bool can_swap;
2819 5284977 : bool can_swap_nonmatching;
2820 5284977 : bool *stmt_can_swap;
2821 :
2822 : /* We're skipping certain operands from processing, for example
2823 : outer loop reduction initial defs. */
2824 5284977 : if (skip_args[i])
2825 : {
2826 511618 : children.safe_push (NULL);
2827 5679052 : continue;
2828 : }
2829 :
2830 4773359 : if (oprnd_info->first_dt == vect_uninitialized_def)
2831 : {
2832 : /* COND_EXPR have one too many eventually if the condition
2833 : is a SSA name. */
2834 0 : gcc_assert (i == 3 && nops == 4);
2835 0 : continue;
2836 : }
2837 :
2838 4773359 : if (oprnd_info->first_gs_p)
2839 : {
2840 23729 : gs_scale = oprnd_info->first_gs_info.scale;
2841 23729 : gs_base = oprnd_info->first_gs_info.base;
2842 : }
2843 :
2844 4773359 : if (is_a <bb_vec_info> (vinfo)
2845 1708288 : && oprnd_info->first_dt == vect_internal_def
2846 5688980 : && !oprnd_info->any_pattern)
2847 : {
2848 : /* For BB vectorization, if all defs are the same do not
2849 : bother to continue the build along the single-lane
2850 : graph but use a splat of the scalar value. */
2851 862022 : stmt_vec_info first_def = oprnd_info->def_stmts[0];
2852 929323 : for (j = 1; j < group_size; ++j)
2853 879497 : if (oprnd_info->def_stmts[j] != first_def)
2854 : break;
2855 862022 : if (j == group_size
2856 : /* But avoid doing this for loads where we may be
2857 : able to CSE things, unless the stmt is not
2858 : vectorizable. */
2859 862022 : && (!STMT_VINFO_VECTORIZABLE (first_def)
2860 59405 : || !gimple_vuse (first_def->stmt)))
2861 : {
2862 40444 : if (dump_enabled_p ())
2863 107 : dump_printf_loc (MSG_NOTE, vect_location,
2864 : "Using a splat of the uniform operand %G",
2865 : first_def->stmt);
2866 40444 : oprnd_info->first_dt = vect_external_def;
2867 : }
2868 : }
2869 :
2870 4773359 : if (oprnd_info->first_dt == vect_external_def
2871 4773359 : || oprnd_info->first_dt == vect_constant_def)
2872 : {
2873 1571011 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
2874 : {
2875 : tree op0;
2876 : tree uniform_val = op0 = oprnd_info->ops[0];
2877 : for (j = 1; j < oprnd_info->ops.length (); ++j)
2878 : if (oprnd_info->ops[j]
2879 : && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
2880 : {
2881 : uniform_val = NULL_TREE;
2882 : break;
2883 : }
2884 : if (!uniform_val
2885 : && !can_duplicate_and_interleave_p (vinfo,
2886 : oprnd_info->ops.length (),
2887 : TREE_TYPE (op0)))
2888 : {
2889 : matches[j] = false;
2890 : if (dump_enabled_p ())
2891 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2892 : "Build SLP failed: invalid type of def "
2893 : "for variable-length SLP %T\n", op0);
2894 : goto fail;
2895 : }
2896 : }
2897 1571011 : slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
2898 1571011 : SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
2899 1571011 : oprnd_info->ops = vNULL;
2900 1571011 : children.safe_push (invnode);
2901 1571011 : continue;
2902 1571011 : }
2903 :
2904 : /* See which SLP operand a reduction chain continues on. We want
2905 : to chain even PHIs but not backedges. */
2906 3202348 : if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
2907 3202348 : || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
2908 : {
2909 237631 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
2910 : {
2911 776 : if (oprnd_info->first_dt == vect_double_reduction_def)
2912 388 : reduc_idx = i;
2913 : }
2914 236855 : else if (is_a <gphi *> (stmt_info->stmt)
2915 236855 : && gimple_phi_num_args
2916 101710 : (as_a <gphi *> (stmt_info->stmt)) != 1)
2917 : ;
2918 135538 : else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
2919 393 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
2920 : ;
2921 135538 : else if (reduc_idx == -1)
2922 127047 : reduc_idx = i;
2923 : else
2924 : /* For .COND_* reduction operations the else value can be the
2925 : same as one of the operation operands. The other def
2926 : stmts have been moved, so we can't check easily. Check
2927 : it's a call at least. */
2928 8491 : gcc_assert (is_a <gcall *> (stmt_info->stmt));
2929 : }
2930 :
2931 : /* When we have a masked load with uniform mask discover this
2932 : as a single-lane mask with a splat permute. This way we can
2933 : recognize this as a masked load-lane by stripping the splat. */
2934 3202348 : if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
2935 58336 : && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
2936 : IFN_MASK_LOAD)
2937 6193 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2938 3202425 : && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
2939 : {
2940 35 : vec<stmt_vec_info> def_stmts2;
2941 35 : def_stmts2.create (1);
2942 35 : def_stmts2.quick_push (oprnd_info->def_stmts[0]);
2943 35 : child = vect_build_slp_tree (vinfo, def_stmts2,
2944 : &this_max_nunits,
2945 : matches, limit,
2946 : &this_tree_size, bst_map);
2947 35 : if (child)
2948 : {
2949 35 : slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
2950 35 : SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
2951 35 : SLP_TREE_LANES (pnode) = group_size;
2952 35 : SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
2953 35 : SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
2954 210 : for (unsigned k = 0; k < group_size; ++k)
2955 : {
2956 175 : SLP_TREE_SCALAR_STMTS (pnode)
2957 175 : .quick_push (oprnd_info->def_stmts[0]);
2958 175 : SLP_TREE_LANE_PERMUTATION (pnode)
2959 175 : .quick_push (std::make_pair (0u, 0u));
2960 : }
2961 35 : SLP_TREE_CHILDREN (pnode).quick_push (child);
2962 35 : pnode->max_nunits = child->max_nunits;
2963 35 : children.safe_push (pnode);
2964 35 : oprnd_info->def_stmts = vNULL;
2965 35 : continue;
2966 35 : }
2967 : else
2968 0 : def_stmts2.release ();
2969 : }
2970 :
2971 6404626 : can_swap = (i == 0
2972 2365163 : && (nops == 2 || nops == 3)
2973 1533629 : && oprnds_info.length () > 1
2974 1533629 : && oprnds_info[1]->first_dt == vect_internal_def
2975 627362 : && (is_gimple_assign (stmt_info->stmt)
2976 51523 : || is_gimple_call (stmt_info->stmt))
2977 : /* Swapping operands for reductions breaks assumptions
2978 : later on. */
2979 3783308 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
2980 3202313 : can_swap_nonmatching = can_swap;
2981 3202313 : stmt_can_swap = NULL;
2982 3202313 : if (can_swap)
2983 : {
2984 525414 : stmt_can_swap = XALLOCAVEC (bool, group_size);
2985 8227127 : for (j = 0; j < group_size; ++j)
2986 : {
2987 7701713 : stmt_can_swap[j] = false;
2988 7701713 : if (!stmts[j])
2989 : /* NULL lanes are gaps and have no stmt to swap. */
2990 0 : stmt_can_swap[j] = true;
2991 7701713 : else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
2992 : {
2993 7696115 : tree_code code = gimple_assign_rhs_code (stmt);
2994 15392230 : stmt_can_swap[j] = (commutative_tree_code (code)
2995 7696115 : || commutative_ternary_tree_code (code));
2996 : }
2997 5598 : else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
2998 : {
2999 5598 : internal_fn fn = (gimple_call_internal_p (call)
3000 5598 : ? gimple_call_internal_fn (call) : IFN_LAST);
3001 11196 : stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
3002 5282 : || commutative_ternary_fn_p (fn))
3003 5634 : && first_commutative_argument (fn) == 0);
3004 : }
3005 :
3006 7701713 : if (j != 0 && !stmt_can_swap[j])
3007 7701713 : can_swap_nonmatching = false;
3008 : }
3009 : }
3010 :
3011 3202313 : old_swap_distance = least_upthread_swappable_op_distance;
3012 3202313 : if (can_swap_nonmatching)
3013 483274 : least_upthread_swappable_op_distance = 1;
3014 2719039 : else if (least_upthread_swappable_op_distance != -1U)
3015 352641 : least_upthread_swappable_op_distance++;
3016 3202313 : child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3017 : &this_max_nunits,
3018 : matches, limit,
3019 : &this_tree_size, bst_map);
3020 3202313 : least_upthread_swappable_op_distance = old_swap_distance;
3021 3202313 : if (child != NULL)
3022 : {
3023 2688327 : oprnd_info->def_stmts = vNULL;
3024 2688327 : children.safe_push (child);
3025 2688327 : continue;
3026 : }
3027 :
3028 : /* If the SLP build for operand zero failed and operand zero
3029 : and one can be commuted try that for the scalar stmts
3030 : that failed the match. */
3031 513986 : if (/* A first scalar stmt mismatch signals a fatal mismatch. */
3032 513986 : matches[0]
3033 291625 : && can_swap)
3034 : {
3035 : /* See whether we can swap the matching or the non-matching
3036 : stmt operands. */
3037 : bool swap_not_matching = true;
3038 76444 : do
3039 : {
3040 7138286 : for (j = 0; j < group_size; ++j)
3041 : {
3042 7078947 : if (matches[j] != !swap_not_matching)
3043 99292 : continue;
3044 : /* Verify if we can swap operands of this stmt. */
3045 6979655 : if (!stmt_can_swap[j])
3046 : {
3047 17105 : if (!swap_not_matching)
3048 7887 : goto fail;
3049 : swap_not_matching = false;
3050 : break;
3051 : }
3052 : }
3053 : }
3054 68557 : while (j != group_size);
3055 :
3056 : /* Swap mismatched definition stmts. */
3057 59339 : if (dump_enabled_p ())
3058 397 : dump_printf_loc (MSG_NOTE, vect_location,
3059 : "Re-trying with swapped operands of stmts ");
3060 7110784 : for (j = 0; j < group_size; ++j)
3061 7051445 : if (matches[j] == !swap_not_matching)
3062 : {
3063 13924688 : std::swap (oprnds_info[0]->def_stmts[j],
3064 6962344 : oprnds_info[1]->def_stmts[j]);
3065 13924688 : std::swap (oprnds_info[0]->ops[j],
3066 6962344 : oprnds_info[1]->ops[j]);
3067 6962344 : if (dump_enabled_p ())
3068 1090 : dump_printf (MSG_NOTE, "%d ", j);
3069 : }
3070 59339 : if (dump_enabled_p ())
3071 397 : dump_printf (MSG_NOTE, "\n");
3072 : /* After swapping some operands we lost track whether an
3073 : operand has any pattern defs so be conservative here. */
3074 115518 : if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
3075 4263 : oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
3076 : /* And try again with scratch 'matches' ... */
3077 59339 : bool *tem = XALLOCAVEC (bool, group_size);
3078 59339 : if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3079 : &this_max_nunits,
3080 : tem, limit,
3081 : &this_tree_size, bst_map)) != NULL)
3082 : {
3083 7798 : oprnd_info->def_stmts = vNULL;
3084 7798 : children.safe_push (child);
3085 7798 : continue;
3086 : }
3087 : }
3088 446760 : fail:
3089 :
3090 : /* If the SLP build failed and we analyze a basic-block
3091 : simply treat nodes we fail to build as externally defined
3092 : (and thus build vectors from the scalar defs).
3093 : The cost model will reject outright expensive cases.
3094 : ??? This doesn't treat cases where permutation ultimatively
3095 : fails (or we don't try permutation below). Ideally we'd
3096 : even compute a permutation that will end up with the maximum
3097 : SLP tree size... */
3098 506188 : if (is_a <bb_vec_info> (vinfo)
3099 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3100 : do extra work to cancel the pattern so the uses see the
3101 : scalar version. */
3102 : /* Skip building vector operands from scalars while operand
3103 : discovery may still be fixed by retrying with swapped operands. */
3104 447407 : && (least_upthread_swappable_op_distance != 1
3105 : /* A first scalar stmt mismatch signals a fatal mismatch
3106 : that the parent commutative retry cannot recover. */
3107 31356 : || !matches[0])
3108 425018 : && !is_pattern_stmt_p (stmt_info)
3109 902660 : && !oprnd_info->any_pattern)
3110 : {
3111 : /* But if there's a leading vector sized set of matching stmts
3112 : fail here so we can split the group. This matches the condition
3113 : vect_analyze_slp_instance uses. */
3114 : /* ??? We might want to split here and combine the results to support
3115 : multiple vector sizes better. */
3116 621317 : for (j = 0; j < group_size; ++j)
3117 621317 : if (!matches[j])
3118 : break;
3119 396174 : if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
3120 396143 : && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
3121 : {
3122 388645 : if (dump_enabled_p ())
3123 665 : dump_printf_loc (MSG_NOTE, vect_location,
3124 : "Building vector operands from scalars\n");
3125 388645 : this_tree_size++;
3126 388645 : child = vect_create_new_slp_node (oprnd_info->ops);
3127 388645 : children.safe_push (child);
3128 388645 : oprnd_info->ops = vNULL;
3129 388645 : continue;
3130 : }
3131 : }
3132 :
3133 117543 : gcc_assert (child == NULL);
3134 138278 : FOR_EACH_VEC_ELT (children, j, child)
3135 20735 : if (child)
3136 20735 : vect_free_slp_tree (child);
3137 117543 : vect_free_oprnd_info (oprnds_info);
3138 117543 : return NULL;
3139 : }
3140 :
3141 3171281 : vect_free_oprnd_info (oprnds_info);
3142 :
3143 : /* If we have all children of a child built up from uniform scalars
3144 : or does more than one possibly expensive vector construction then
3145 : just throw that away, causing it built up from scalars.
3146 : The exception is the SLP node for the vector store. */
3147 3171281 : if (is_a <bb_vec_info> (vinfo)
3148 1156606 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
3149 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3150 : do extra work to cancel the pattern so the uses see the
3151 : scalar version. */
3152 3656576 : && !is_pattern_stmt_p (stmt_info))
3153 : {
3154 : slp_tree child;
3155 : unsigned j;
3156 : bool all_uniform_p = true;
3157 : unsigned n_vector_builds = 0;
3158 1353685 : FOR_EACH_VEC_ELT (children, j, child)
3159 : {
3160 902816 : if (!child)
3161 : ;
3162 902816 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
3163 : all_uniform_p = false;
3164 644688 : else if (!vect_slp_tree_uniform_p (child))
3165 : {
3166 486496 : all_uniform_p = false;
3167 486496 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
3168 445919 : n_vector_builds++;
3169 : }
3170 : }
3171 450869 : if (all_uniform_p
3172 450869 : || n_vector_builds > 1
3173 766873 : || (n_vector_builds == children.length ()
3174 35008 : && is_a <gphi *> (stmt_info->stmt)))
3175 : {
3176 : /* Roll back. */
3177 139836 : matches[0] = false;
3178 442655 : FOR_EACH_VEC_ELT (children, j, child)
3179 302819 : if (child)
3180 302819 : vect_free_slp_tree (child);
3181 :
3182 139836 : if (dump_enabled_p ())
3183 225 : dump_printf_loc (MSG_NOTE, vect_location,
3184 : "Building parent vector operands from "
3185 : "scalars instead\n");
3186 139836 : return NULL;
3187 : }
3188 : }
3189 :
3190 3031445 : *tree_size += this_tree_size + 1;
3191 3031445 : *max_nunits = this_max_nunits;
3192 :
3193 3031445 : if (two_operators)
3194 : {
3195 : /* ??? We'd likely want to either cache in bst_map sth like
3196 : { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
3197 : the true { a+b, a+b, a+b, a+b } ... but there we don't have
3198 : explicit stmts to put in so the keying on 'stmts' doesn't
3199 : work (but we have the same issue with nodes that use 'ops'). */
3200 :
3201 7560 : if (has_two_operators_perm)
3202 : {
3203 46 : slp_tree child = children[0];
3204 46 : children.truncate (0);
3205 138 : for (i = 0; i < 2; i++)
3206 : {
3207 92 : slp_tree pnode
3208 92 : = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
3209 92 : SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
3210 92 : SLP_TREE_VECTYPE (pnode) = vectype;
3211 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3212 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3213 92 : lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
3214 92 : children.safe_push (pnode);
3215 :
3216 716 : for (unsigned j = 0; j < stmts.length (); j++)
3217 624 : perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
3218 : }
3219 :
3220 46 : SLP_TREE_REF_COUNT (child) += 4;
3221 : }
3222 :
3223 7560 : slp_tree one = new _slp_tree;
3224 7560 : slp_tree two = new _slp_tree;
3225 7560 : SLP_TREE_DEF_TYPE (one) = vect_internal_def;
3226 7560 : SLP_TREE_DEF_TYPE (two) = vect_internal_def;
3227 7560 : SLP_TREE_VECTYPE (one) = vectype;
3228 7560 : SLP_TREE_VECTYPE (two) = vectype;
3229 7560 : SLP_TREE_CHILDREN (one).safe_splice (children);
3230 7560 : SLP_TREE_CHILDREN (two).safe_splice (children);
3231 7560 : slp_tree child;
3232 30242 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
3233 15122 : SLP_TREE_REF_COUNT (child)++;
3234 :
3235 : /* Here we record the original defs since this
3236 : node represents the final lane configuration. */
3237 7560 : node = vect_create_new_slp_node (node, stmts, 2);
3238 7560 : SLP_TREE_VECTYPE (node) = vectype;
3239 7560 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
3240 7560 : SLP_TREE_CHILDREN (node).quick_push (one);
3241 7560 : SLP_TREE_CHILDREN (node).quick_push (two);
3242 7560 : enum tree_code code0 = ERROR_MARK;
3243 7560 : enum tree_code ocode = ERROR_MARK;
3244 7560 : if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
3245 7558 : code0 = gimple_assign_rhs_code (stmt);
3246 7560 : stmt_vec_info ostmt_info;
3247 7560 : unsigned j = 0;
3248 27065 : FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
3249 : {
3250 19505 : int op = 0;
3251 19505 : if (gassign *ostmt = dyn_cast <gassign *> (ostmt_info->stmt))
3252 : {
3253 19501 : if (gimple_assign_rhs_code (ostmt) != code0)
3254 : {
3255 9771 : ocode = gimple_assign_rhs_code (ostmt);
3256 : op = 1;
3257 : j = i;
3258 : }
3259 : }
3260 : else
3261 : {
3262 8 : if (gimple_call_combined_fn (stmts[0]->stmt)
3263 4 : != gimple_call_combined_fn (ostmt_info->stmt))
3264 : {
3265 2 : op = 1;
3266 2 : j = i;
3267 : }
3268 : }
3269 19505 : SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
3270 : }
3271 7560 : SLP_TREE_CODE (one) = code0;
3272 7560 : SLP_TREE_CODE (two) = ocode;
3273 7560 : SLP_TREE_LANES (one) = stmts.length ();
3274 7560 : SLP_TREE_LANES (two) = stmts.length ();
3275 7560 : SLP_TREE_REPRESENTATIVE (one) = stmts[0];
3276 7560 : SLP_TREE_REPRESENTATIVE (two) = stmts[j];
3277 :
3278 7560 : return node;
3279 : }
3280 :
3281 3023885 : node = vect_create_new_slp_node (node, stmts, nops);
3282 3023885 : SLP_TREE_VECTYPE (node) = vectype;
3283 3023885 : SLP_TREE_CHILDREN (node).splice (children);
3284 3023885 : SLP_TREE_GS_SCALE (node) = gs_scale;
3285 3023885 : SLP_TREE_GS_BASE (node) = gs_base;
3286 3023885 : if (reduc_idx != -1)
3287 : {
3288 118928 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
3289 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3290 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
3291 118928 : SLP_TREE_REDUC_IDX (node) = reduc_idx;
3292 118928 : node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
3293 : }
3294 : /* When reaching the reduction PHI, create a vect_reduc_info. */
3295 2904957 : else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
3296 2904957 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3297 2904957 : && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
3298 : {
3299 103681 : loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
3300 103681 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3301 103681 : node->cycle_info.id = loop_vinfo->reduc_infos.length ();
3302 103681 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
3303 103681 : loop_vinfo->reduc_infos.safe_push (reduc_info);
3304 103681 : stmt_vec_info reduc_phi = stmt_info;
3305 : /* ??? For double reductions vect_is_simple_reduction stores the
3306 : reduction type and code on the inner loop header PHI. */
3307 103681 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3308 : {
3309 388 : use_operand_p use_p;
3310 388 : gimple *use_stmt;
3311 388 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
3312 : &use_p, &use_stmt);
3313 388 : gcc_assert (res);
3314 388 : reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
3315 : }
3316 103681 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
3317 103681 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
3318 103681 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
3319 103681 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
3320 : }
3321 : return node;
3322 9866472 : }
3323 :
3324 : /* Dump a single SLP tree NODE. */
3325 :
3326 : static void
3327 457372 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
3328 : slp_tree node)
3329 : {
3330 457372 : unsigned i, j;
3331 457372 : slp_tree child;
3332 457372 : stmt_vec_info stmt_info;
3333 457372 : tree op;
3334 :
3335 457372 : dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
3336 457372 : dump_user_location_t user_loc = loc.get_user_location ();
3337 457372 : dump_printf_loc (metadata, user_loc,
3338 : "node%s %p (max_nunits=" HOST_WIDE_INT_PRINT_UNSIGNED
3339 : ", refcnt=%u)",
3340 457372 : SLP_TREE_DEF_TYPE (node) == vect_external_def
3341 : ? " (external)"
3342 : : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
3343 441185 : ? " (constant)"
3344 : : ""), (void *) node,
3345 457372 : estimated_poly_value (node->max_nunits),
3346 : SLP_TREE_REF_COUNT (node));
3347 457372 : if (SLP_TREE_VECTYPE (node))
3348 387049 : dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
3349 457372 : dump_printf (metadata, "%s",
3350 457372 : node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
3351 457372 : if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
3352 24342 : dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
3353 : node->cycle_info.reduc_idx);
3354 457372 : dump_printf (metadata, "\n");
3355 457372 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3356 : {
3357 371909 : if (SLP_TREE_PERMUTE_P (node))
3358 13902 : dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
3359 : else
3360 358007 : dump_printf_loc (metadata, user_loc, "op template: %G",
3361 358007 : SLP_TREE_REPRESENTATIVE (node)->stmt);
3362 : }
3363 457372 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
3364 892146 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
3365 528456 : if (stmt_info)
3366 523058 : dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
3367 523058 : SLP_TREE_LIVE_LANES (node).contains (i)
3368 519376 : ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
3369 519376 : ? "[l] " : ""),
3370 : i, stmt_info->stmt);
3371 : else
3372 5398 : dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
3373 : else
3374 : {
3375 93682 : dump_printf_loc (metadata, user_loc, "\t{ ");
3376 206648 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
3377 112966 : dump_printf (metadata, "%T%s ", op,
3378 112966 : i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
3379 93682 : dump_printf (metadata, "}\n");
3380 : }
3381 457372 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
3382 : {
3383 66453 : dump_printf_loc (metadata, user_loc, "\tload permutation {");
3384 151983 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
3385 85530 : dump_printf (dump_kind, " %u", j);
3386 66453 : dump_printf (dump_kind, " }\n");
3387 : }
3388 457372 : if (SLP_TREE_LANE_PERMUTATION (node).exists ())
3389 : {
3390 13910 : dump_printf_loc (metadata, user_loc, "\tlane permutation {");
3391 52215 : for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
3392 38305 : dump_printf (dump_kind, " %u[%u]",
3393 38305 : SLP_TREE_LANE_PERMUTATION (node)[i].first,
3394 38305 : SLP_TREE_LANE_PERMUTATION (node)[i].second);
3395 13910 : dump_printf (dump_kind, " }%s\n",
3396 13910 : node->ldst_lanes ? " (load-lanes)" : "");
3397 : }
3398 457372 : if (SLP_TREE_CHILDREN (node).is_empty ())
3399 174227 : return;
3400 283145 : dump_printf_loc (metadata, user_loc, "\tchildren");
3401 746776 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3402 463631 : dump_printf (dump_kind, " %p", (void *)child);
3403 283145 : dump_printf (dump_kind, "%s\n",
3404 283145 : node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
3405 : ? " (store-lanes)" : "");
3406 : }
3407 :
3408 : DEBUG_FUNCTION void
3409 0 : debug (slp_tree node)
3410 : {
3411 0 : debug_dump_context ctx;
3412 0 : vect_print_slp_tree (MSG_NOTE,
3413 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3414 : node);
3415 0 : }
3416 :
3417 : /* Recursive helper for the dot producer below. */
3418 :
3419 : static void
3420 0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
3421 : {
3422 0 : if (visited.add (node))
3423 : return;
3424 :
3425 0 : fprintf (f, "\"%p\" [label=\"", (void *)node);
3426 0 : vect_print_slp_tree (MSG_NOTE,
3427 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3428 : node);
3429 0 : fprintf (f, "\"];\n");
3430 :
3431 :
3432 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3433 0 : fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
3434 :
3435 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3436 0 : if (child)
3437 0 : dot_slp_tree (f, child, visited);
3438 : }
3439 :
3440 : DEBUG_FUNCTION void
3441 0 : dot_slp_tree (const char *fname, slp_tree node)
3442 : {
3443 0 : FILE *f = fopen (fname, "w");
3444 0 : fprintf (f, "digraph {\n");
3445 0 : fflush (f);
3446 0 : {
3447 0 : debug_dump_context ctx (f);
3448 0 : hash_set<slp_tree> visited;
3449 0 : dot_slp_tree (f, node, visited);
3450 0 : }
3451 0 : fflush (f);
3452 0 : fprintf (f, "}\n");
3453 0 : fclose (f);
3454 0 : }
3455 :
3456 : DEBUG_FUNCTION void
3457 0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
3458 : {
3459 0 : FILE *f = fopen (fname, "w");
3460 0 : fprintf (f, "digraph {\n");
3461 0 : fflush (f);
3462 0 : {
3463 0 : debug_dump_context ctx (f);
3464 0 : hash_set<slp_tree> visited;
3465 0 : for (auto inst : slp_instances)
3466 0 : dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
3467 0 : }
3468 0 : fflush (f);
3469 0 : fprintf (f, "}\n");
3470 0 : fclose (f);
3471 0 : }
3472 :
3473 : /* Dump a slp tree NODE using flags specified in DUMP_KIND. */
3474 :
3475 : static void
3476 496730 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3477 : slp_tree node, hash_set<slp_tree> &visited)
3478 : {
3479 496730 : unsigned i;
3480 496730 : slp_tree child;
3481 :
3482 496730 : if (visited.add (node))
3483 496730 : return;
3484 :
3485 456899 : vect_print_slp_tree (dump_kind, loc, node);
3486 :
3487 1376916 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3488 463118 : if (child)
3489 419301 : vect_print_slp_graph (dump_kind, loc, child, visited);
3490 : }
3491 :
3492 : static void
3493 47399 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3494 : slp_tree entry)
3495 : {
3496 47399 : hash_set<slp_tree> visited;
3497 47399 : vect_print_slp_graph (dump_kind, loc, entry, visited);
3498 47399 : }
3499 :
3500 : DEBUG_FUNCTION void
3501 0 : debug (slp_instance instance)
3502 : {
3503 0 : debug_dump_context ctx;
3504 0 : vect_print_slp_graph (MSG_NOTE,
3505 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3506 : SLP_INSTANCE_TREE (instance));
3507 0 : }
3508 :
3509 :
3510 : /* Compute the set of scalar stmts participating in external nodes. */
3511 :
3512 : static void
3513 1688819 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
3514 : hash_set<slp_tree> &visited,
3515 : hash_set<stmt_vec_info> &estmts)
3516 : {
3517 1688819 : if (visited.add (node))
3518 : return;
3519 :
3520 1629876 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3521 : {
3522 : slp_tree child;
3523 : int i;
3524 1908994 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3525 975684 : if (child)
3526 975684 : vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
3527 : }
3528 : else
3529 3897814 : for (tree def : SLP_TREE_SCALAR_OPS (node))
3530 : {
3531 1810118 : stmt_vec_info def_stmt = vinfo->lookup_def (def);
3532 1810118 : if (def_stmt)
3533 388280 : estmts.add (def_stmt);
3534 : }
3535 : }
3536 :
3537 : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
3538 : with STMT_SLP_TYPE == pure_slp. */
3539 :
3540 : static void
3541 245217 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
3542 : {
3543 : /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
3544 : walk on. */
3545 245217 : hash_set<stmt_vec_info> scalar_stmts_in_externs;
3546 245217 : hash_set<slp_tree> visited;
3547 1448786 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3548 713135 : vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
3549 : visited, scalar_stmts_in_externs);
3550 :
3551 : /* DFS walk scalar stmts to compute the vectorized coverage indicated
3552 : by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
3553 : stmts. */
3554 1448786 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3555 : {
3556 901805 : for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
3557 86680 : if (!scalar_stmts_in_externs.contains (stmt))
3558 85284 : STMT_SLP_TYPE (stmt) = pure_slp;
3559 713135 : auto_vec<stmt_vec_info> worklist;
3560 4017815 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
3561 : {
3562 1878410 : stmt = vect_orig_stmt (stmt);
3563 1878410 : if (!scalar_stmts_in_externs.contains (stmt)
3564 1878410 : && STMT_SLP_TYPE (stmt) != pure_slp)
3565 : {
3566 1863085 : STMT_SLP_TYPE (stmt) = pure_slp;
3567 1863085 : worklist.safe_push (stmt);
3568 : }
3569 : }
3570 3755662 : while (!worklist.is_empty ())
3571 : {
3572 2333478 : stmt_vec_info stmt = worklist.pop ();
3573 :
3574 : /* Now walk relevant parts of the SSA use-def graph. */
3575 2333478 : slp_oprnds child_ops (stmt);
3576 4950627 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
3577 : {
3578 2617149 : tree op = child_ops.get_op_for_slp_child (stmt, i);
3579 2617149 : stmt_vec_info def = vinfo->lookup_def (op);
3580 2617149 : if (def
3581 943739 : && !scalar_stmts_in_externs.contains (def)
3582 3183383 : && STMT_SLP_TYPE (def) != pure_slp)
3583 : {
3584 470393 : STMT_SLP_TYPE (def) = pure_slp;
3585 470393 : worklist.safe_push (def);
3586 : }
3587 : }
3588 : }
3589 713135 : }
3590 245217 : }
3591 :
3592 : /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
3593 :
3594 : static void
3595 2592623 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
3596 : {
3597 2592623 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3598 : return;
3599 :
3600 1555580 : if (visited.add (node))
3601 : return;
3602 :
3603 7257091 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
3604 3237782 : if (stmt_info)
3605 : {
3606 3237782 : gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
3607 : || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
3608 3237782 : STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
3609 : }
3610 :
3611 5637180 : for (auto child: SLP_TREE_CHILDREN (node))
3612 1775213 : if (child)
3613 1775213 : vect_mark_slp_stmts_relevant (child, visited);
3614 : }
3615 :
3616 : static void
3617 817410 : vect_mark_slp_stmts_relevant (slp_tree node)
3618 : {
3619 817410 : hash_set<slp_tree> visited;
3620 817410 : vect_mark_slp_stmts_relevant (node, visited);
3621 817410 : }
3622 :
3623 :
3624 : /* Gather loads in the SLP graph NODE and populate the INST loads array. */
3625 :
3626 : static void
3627 11121276 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
3628 : hash_set<slp_tree> &visited)
3629 : {
3630 11121276 : if (!node || visited.add (node))
3631 1822818 : return;
3632 :
3633 9298458 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3634 : return;
3635 :
3636 6873716 : if (!SLP_TREE_PERMUTE_P (node))
3637 : {
3638 6661106 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
3639 6661106 : if (STMT_VINFO_DATA_REF (stmt_info)
3640 2853477 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
3641 1625913 : loads.safe_push (node);
3642 : }
3643 :
3644 : unsigned i;
3645 : slp_tree child;
3646 15677545 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3647 8803829 : vect_gather_slp_loads (loads, child, visited);
3648 : }
3649 :
3650 :
3651 : /* Find the last store in SLP INSTANCE. */
3652 :
3653 : stmt_vec_info
3654 2821234 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
3655 : {
3656 2821234 : stmt_vec_info last = NULL;
3657 15884282 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3658 7431310 : if (stmt_vinfo)
3659 : {
3660 7431310 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3661 7431310 : last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
3662 : }
3663 :
3664 2821234 : return last;
3665 : }
3666 :
3667 : /* Find the first stmt in NODE. */
3668 :
3669 : stmt_vec_info
3670 565434 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
3671 : {
3672 565434 : stmt_vec_info first = NULL;
3673 :
3674 3051892 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3675 1355590 : if (stmt_vinfo)
3676 : {
3677 1352894 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3678 1352894 : if (!first
3679 1352894 : || get_later_stmt (stmt_vinfo, first) == first)
3680 : first = stmt_vinfo;
3681 : }
3682 :
3683 565434 : return first;
3684 : }
3685 :
3686 : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
3687 : two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
3688 : (also containing the first GROUP1_SIZE stmts, since stores are
3689 : consecutive), the second containing the remainder.
3690 : Return the first stmt in the second group. */
3691 :
3692 : static stmt_vec_info
3693 161882 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
3694 : {
3695 161882 : gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
3696 161882 : gcc_assert (group1_size > 0);
3697 161882 : int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
3698 161882 : gcc_assert (group2_size > 0);
3699 161882 : DR_GROUP_SIZE (first_vinfo) = group1_size;
3700 :
3701 161882 : stmt_vec_info stmt_info = first_vinfo;
3702 541340 : for (unsigned i = group1_size; i > 1; i--)
3703 : {
3704 379458 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
3705 379458 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3706 : }
3707 : /* STMT is now the last element of the first group. */
3708 161882 : stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
3709 161882 : DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
3710 :
3711 161882 : DR_GROUP_SIZE (group2) = group2_size;
3712 451173 : for (stmt_info = group2; stmt_info;
3713 289291 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
3714 : {
3715 289291 : DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
3716 289291 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3717 : }
3718 :
3719 : /* For the second group, the DR_GROUP_GAP is that before the original group,
3720 : plus skipping over the first vector. */
3721 161882 : DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
3722 :
3723 : /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
3724 161882 : DR_GROUP_GAP (first_vinfo) += group2_size;
3725 :
3726 161882 : if (dump_enabled_p ())
3727 70 : dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
3728 : group1_size, group2_size);
3729 :
3730 161882 : return group2;
3731 : }
3732 :
3733 : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
3734 : statements and a vector of NUNITS elements. */
3735 :
3736 : static poly_uint64
3737 4367416 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
3738 : {
3739 4367416 : return exact_div (common_multiple (nunits, group_size), group_size);
3740 : }
3741 :
3742 : /* Helper that checks to see if a node is a load node. */
3743 :
3744 : static inline bool
3745 103 : vect_is_slp_load_node (slp_tree root)
3746 : {
3747 103 : return (!SLP_TREE_PERMUTE_P (root)
3748 103 : && SLP_TREE_DEF_TYPE (root) == vect_internal_def
3749 97 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
3750 167 : && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
3751 : }
3752 :
3753 :
3754 : /* Helper function of optimize_load_redistribution that performs the operation
3755 : recursively. */
3756 :
3757 : static slp_tree
3758 21783 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
3759 : vec_info *vinfo, unsigned int group_size,
3760 : hash_map<slp_tree, slp_tree> *load_map,
3761 : slp_tree root)
3762 : {
3763 21783 : if (slp_tree *leader = load_map->get (root))
3764 4549 : return *leader;
3765 :
3766 17234 : slp_tree node;
3767 17234 : unsigned i;
3768 :
3769 : /* For now, we don't know anything about externals so do not do anything. */
3770 17234 : if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
3771 : return NULL;
3772 12109 : else if (SLP_TREE_PERMUTE_P (root))
3773 : {
3774 : /* First convert this node into a load node and add it to the leaves
3775 : list and flatten the permute from a lane to a load one. If it's
3776 : unneeded it will be elided later. */
3777 71 : vec<stmt_vec_info> stmts;
3778 71 : stmts.create (SLP_TREE_LANES (root));
3779 71 : lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
3780 135 : for (unsigned j = 0; j < lane_perm.length (); j++)
3781 : {
3782 103 : std::pair<unsigned, unsigned> perm = lane_perm[j];
3783 103 : node = SLP_TREE_CHILDREN (root)[perm.first];
3784 :
3785 103 : if (!vect_is_slp_load_node (node)
3786 103 : || SLP_TREE_CHILDREN (node).exists ())
3787 : {
3788 39 : stmts.release ();
3789 39 : goto next;
3790 : }
3791 :
3792 64 : stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
3793 : }
3794 :
3795 32 : if (dump_enabled_p ())
3796 0 : dump_printf_loc (MSG_NOTE, vect_location,
3797 : "converting stmts on permute node %p\n",
3798 : (void *) root);
3799 :
3800 32 : bool *matches = XALLOCAVEC (bool, group_size);
3801 32 : poly_uint64 max_nunits = 1;
3802 32 : unsigned tree_size = 0, limit = 1;
3803 32 : node = vect_build_slp_tree (vinfo, stmts, &max_nunits,
3804 : matches, &limit, &tree_size, bst_map);
3805 32 : if (!node)
3806 0 : stmts.release ();
3807 :
3808 32 : load_map->put (root, node);
3809 32 : return node;
3810 : }
3811 :
3812 12038 : next:
3813 12077 : load_map->put (root, NULL);
3814 :
3815 27503 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3816 : {
3817 15426 : slp_tree value
3818 15426 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3819 : node);
3820 15426 : if (value)
3821 : {
3822 32 : SLP_TREE_REF_COUNT (value)++;
3823 32 : SLP_TREE_CHILDREN (root)[i] = value;
3824 : /* ??? We know the original leafs of the replaced nodes will
3825 : be referenced by bst_map, only the permutes created by
3826 : pattern matching are not. */
3827 32 : if (SLP_TREE_REF_COUNT (node) == 1)
3828 32 : load_map->remove (node);
3829 32 : vect_free_slp_tree (node);
3830 : }
3831 : }
3832 :
3833 : return NULL;
3834 : }
3835 :
3836 : /* Temporary workaround for loads not being CSEd during SLP build. This
3837 : function will traverse the SLP tree rooted in ROOT for INSTANCE and find
3838 : VEC_PERM nodes that blend vectors from multiple nodes that all read from the
3839 : same DR such that the final operation is equal to a permuted load. Such
3840 : NODES are then directly converted into LOADS themselves. The nodes are
3841 : CSEd using BST_MAP. */
3842 :
3843 : static void
3844 4175 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
3845 : vec_info *vinfo, unsigned int group_size,
3846 : hash_map<slp_tree, slp_tree> *load_map,
3847 : slp_tree root)
3848 : {
3849 4175 : slp_tree node;
3850 4175 : unsigned i;
3851 :
3852 10532 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3853 : {
3854 6357 : slp_tree value
3855 6357 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3856 : node);
3857 6357 : if (value)
3858 : {
3859 0 : SLP_TREE_REF_COUNT (value)++;
3860 0 : SLP_TREE_CHILDREN (root)[i] = value;
3861 : /* ??? We know the original leafs of the replaced nodes will
3862 : be referenced by bst_map, only the permutes created by
3863 : pattern matching are not. */
3864 0 : if (SLP_TREE_REF_COUNT (node) == 1)
3865 0 : load_map->remove (node);
3866 0 : vect_free_slp_tree (node);
3867 : }
3868 : }
3869 4175 : }
3870 :
3871 : /* Helper function of vect_match_slp_patterns.
3872 :
3873 : Attempts to match patterns against the slp tree rooted in REF_NODE using
3874 : VINFO. Patterns are matched in post-order traversal.
3875 :
3876 : If matching is successful the value in REF_NODE is updated and returned, if
3877 : not then it is returned unchanged. */
3878 :
3879 : static bool
3880 6468018 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
3881 : slp_tree_to_load_perm_map_t *perm_cache,
3882 : slp_compat_nodes_map_t *compat_cache,
3883 : hash_set<slp_tree> *visited)
3884 : {
3885 6468018 : unsigned i;
3886 6468018 : slp_tree node = *ref_node;
3887 6468018 : bool found_p = false;
3888 6468018 : if (!node || visited->add (node))
3889 922690 : return false;
3890 :
3891 : slp_tree child;
3892 10395317 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3893 4849989 : found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
3894 : vinfo, perm_cache, compat_cache,
3895 : visited);
3896 :
3897 16635984 : for (unsigned x = 0; x < num__slp_patterns; x++)
3898 : {
3899 11090656 : vect_pattern *pattern
3900 11090656 : = slp_patterns[x] (perm_cache, compat_cache, ref_node);
3901 11090656 : if (pattern)
3902 : {
3903 1108 : pattern->build (vinfo);
3904 1108 : delete pattern;
3905 1108 : found_p = true;
3906 : }
3907 : }
3908 :
3909 : return found_p;
3910 : }
3911 :
3912 : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
3913 : vec_info VINFO.
3914 :
3915 : The modified tree is returned. Patterns are tried in order and multiple
3916 : patterns may match. */
3917 :
3918 : static bool
3919 1618029 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
3920 : hash_set<slp_tree> *visited,
3921 : slp_tree_to_load_perm_map_t *perm_cache,
3922 : slp_compat_nodes_map_t *compat_cache)
3923 : {
3924 1618029 : DUMP_VECT_SCOPE ("vect_match_slp_patterns");
3925 1618029 : slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
3926 :
3927 1618029 : if (dump_enabled_p ())
3928 31250 : dump_printf_loc (MSG_NOTE, vect_location,
3929 : "Analyzing SLP tree %p for patterns\n",
3930 31250 : (void *) SLP_INSTANCE_TREE (instance));
3931 :
3932 1618029 : return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
3933 1618029 : visited);
3934 : }
3935 :
3936 : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
3937 : vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether
3938 : the stores are masked.
3939 : Return true if we could use IFN_STORE_LANES instead and if that appears
3940 : to be the better approach. */
3941 :
3942 : static bool
3943 6151 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
3944 : tree vectype, bool masked_p,
3945 : unsigned int group_size,
3946 : unsigned int new_group_size)
3947 : {
3948 6151 : if (!vectype)
3949 : {
3950 6151 : tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
3951 6151 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
3952 : }
3953 6151 : if (!vectype)
3954 : return false;
3955 : /* Allow the split if one of the two new groups would operate on full
3956 : vectors *within* rather than across one scalar loop iteration.
3957 : This is purely a heuristic, but it should work well for group
3958 : sizes of 3 and 4, where the possible splits are:
3959 :
3960 : 3->2+1: OK if the vector has exactly two elements
3961 : 4->2+2: Likewise
3962 : 4->3+1: Less clear-cut. */
3963 6151 : if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
3964 3500 : || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
3965 2674 : return false;
3966 3477 : return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
3967 : }
3968 :
3969 : /* Analyze an SLP instance starting from a group of grouped stores. Call
3970 : vect_build_slp_tree to build a tree of packed stmts if possible.
3971 : Return FALSE if it's impossible to SLP any stmt in the loop. */
3972 :
3973 : static bool
3974 : vect_analyze_slp_instance (vec_info *vinfo,
3975 : scalar_stmts_to_slp_tree_map_t *bst_map,
3976 : stmt_vec_info stmt_info, slp_instance_kind kind,
3977 : unsigned max_tree_size, unsigned *limit,
3978 : bool force_single_lane);
3979 :
3980 : /* Build an interleaving scheme for the store sources RHS_NODES from
3981 : SCALAR_STMTS. */
3982 :
3983 : static slp_tree
3984 8242 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
3985 : vec<stmt_vec_info> &scalar_stmts,
3986 : poly_uint64 max_nunits)
3987 : {
3988 8242 : unsigned int group_size = scalar_stmts.length ();
3989 16484 : slp_tree node = vect_create_new_slp_node (scalar_stmts,
3990 8242 : SLP_TREE_CHILDREN
3991 : (rhs_nodes[0]).length ());
3992 8242 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
3993 8242 : node->max_nunits = max_nunits;
3994 8242 : for (unsigned l = 0;
3995 16511 : l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
3996 : {
3997 : /* And a permute merging all RHS SLP trees. */
3998 8269 : slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
3999 8269 : VEC_PERM_EXPR);
4000 8269 : SLP_TREE_CHILDREN (node).quick_push (perm);
4001 8269 : SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
4002 8269 : SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
4003 8269 : perm->max_nunits = max_nunits;
4004 8269 : SLP_TREE_LANES (perm) = group_size;
4005 : /* ??? We should set this NULL but that's not expected. */
4006 8269 : SLP_TREE_REPRESENTATIVE (perm)
4007 8269 : = SLP_TREE_REPRESENTATIVE (SLP_TREE_CHILDREN (rhs_nodes[0])[l]);
4008 32149 : for (unsigned j = 0; j < rhs_nodes.length (); ++j)
4009 : {
4010 23880 : SLP_TREE_CHILDREN (perm)
4011 23880 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
4012 23880 : SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
4013 23880 : for (unsigned k = 0;
4014 50108 : k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
4015 : {
4016 : /* ??? We should populate SLP_TREE_SCALAR_STMTS
4017 : or SLP_TREE_SCALAR_OPS but then we might have
4018 : a mix of both in our children. */
4019 26228 : SLP_TREE_LANE_PERMUTATION (perm)
4020 26228 : .quick_push (std::make_pair (j, k));
4021 : }
4022 : }
4023 :
4024 : /* Now we have a single permute node but we cannot code-generate
4025 : the case with more than two inputs.
4026 : Perform pairwise reduction, reducing the two inputs
4027 : with the least number of lanes to one and then repeat until
4028 : we end up with two inputs. That scheme makes sure we end
4029 : up with permutes satisfying the restriction of requiring at
4030 : most two vector inputs to produce a single vector output
4031 : when the number of lanes is even. */
4032 15611 : while (SLP_TREE_CHILDREN (perm).length () > 2)
4033 : {
4034 : /* When we have three equal sized groups left the pairwise
4035 : reduction does not result in a scheme that avoids using
4036 : three vectors. Instead merge the first two groups
4037 : to the final size with do-not-care elements (chosen
4038 : from the first group) and then merge with the third.
4039 : { A0, B0, x, A1, B1, x, ... }
4040 : -> { A0, B0, C0, A1, B1, C1, ... }
4041 : This handles group size of three (and at least
4042 : power-of-two multiples of that). */
4043 7342 : if (SLP_TREE_CHILDREN (perm).length () == 3
4044 3387 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4045 3387 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
4046 7342 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4047 2515 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
4048 : {
4049 2209 : int ai = 0;
4050 2209 : int bi = 1;
4051 2209 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4052 2209 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4053 2209 : unsigned n = SLP_TREE_LANES (perm);
4054 :
4055 2209 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4056 2209 : SLP_TREE_LANES (permab) = n;
4057 2209 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4058 2209 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4059 2209 : permab->max_nunits = max_nunits;
4060 : /* ??? Should be NULL but that's not expected. */
4061 2209 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4062 2209 : SLP_TREE_CHILDREN (permab).quick_push (a);
4063 4432 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4064 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4065 2223 : .quick_push (std::make_pair (0, k));
4066 2209 : SLP_TREE_CHILDREN (permab).quick_push (b);
4067 4432 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4068 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4069 2223 : .quick_push (std::make_pair (1, k));
4070 : /* Push the do-not-care lanes. */
4071 4432 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4072 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4073 2223 : .quick_push (std::make_pair (0, k));
4074 :
4075 : /* Put the merged node into 'perm', in place of a. */
4076 2209 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4077 : /* Adjust the references to b in the permutation
4078 : of perm and to the later children which we'll
4079 : remove. */
4080 8878 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4081 : {
4082 6669 : std::pair<unsigned, unsigned> &p
4083 6669 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4084 6669 : if (p.first == (unsigned) bi)
4085 : {
4086 2223 : p.first = ai;
4087 2223 : p.second += SLP_TREE_LANES (a);
4088 : }
4089 4446 : else if (p.first > (unsigned) bi)
4090 2223 : p.first--;
4091 : }
4092 2209 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4093 2209 : break;
4094 : }
4095 :
4096 : /* Pick the two nodes with the least number of lanes,
4097 : prefer the earliest candidate and maintain ai < bi. */
4098 : int ai = -1;
4099 : int bi = -1;
4100 46237 : for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
4101 : {
4102 41104 : if (ai == -1)
4103 5133 : ai = ci;
4104 35971 : else if (bi == -1)
4105 5133 : bi = ci;
4106 30838 : else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4107 30838 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
4108 30838 : || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4109 25353 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
4110 : {
4111 11860 : if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
4112 5930 : <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
4113 2730 : bi = ci;
4114 : else
4115 : {
4116 3200 : ai = bi;
4117 3200 : bi = ci;
4118 : }
4119 : }
4120 : }
4121 :
4122 : /* Produce a merge of nodes ai and bi. */
4123 5133 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4124 5133 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4125 5133 : unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
4126 5133 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4127 5133 : SLP_TREE_LANES (permab) = n;
4128 5133 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4129 5133 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4130 5133 : permab->max_nunits = max_nunits;
4131 : /* ??? Should be NULL but that's not expected. */
4132 5133 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4133 5133 : SLP_TREE_CHILDREN (permab).quick_push (a);
4134 13514 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4135 8381 : SLP_TREE_LANE_PERMUTATION (permab)
4136 8381 : .quick_push (std::make_pair (0, k));
4137 5133 : SLP_TREE_CHILDREN (permab).quick_push (b);
4138 12838 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4139 7705 : SLP_TREE_LANE_PERMUTATION (permab)
4140 7705 : .quick_push (std::make_pair (1, k));
4141 :
4142 : /* Put the merged node into 'perm', in place of a. */
4143 5133 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4144 : /* Adjust the references to b in the permutation
4145 : of perm and to the later children which we'll
4146 : remove. */
4147 73687 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4148 : {
4149 68554 : std::pair<unsigned, unsigned> &p
4150 68554 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4151 68554 : if (p.first == (unsigned) bi)
4152 : {
4153 7705 : p.first = ai;
4154 7705 : p.second += SLP_TREE_LANES (a);
4155 : }
4156 60849 : else if (p.first > (unsigned) bi)
4157 25548 : p.first--;
4158 : }
4159 5133 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4160 : }
4161 : }
4162 :
4163 8242 : return node;
4164 : }
4165 :
4166 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4167 : of KIND. Return true if successful. SCALAR_STMTS is owned by this
4168 : function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
4169 : the caller upon failure. */
4170 :
4171 : static bool
4172 2133951 : vect_build_slp_instance (vec_info *vinfo,
4173 : slp_instance_kind kind,
4174 : vec<stmt_vec_info> &scalar_stmts,
4175 : vec<stmt_vec_info> &root_stmt_infos,
4176 : vec<tree> &remain,
4177 : unsigned max_tree_size, unsigned *limit,
4178 : scalar_stmts_to_slp_tree_map_t *bst_map,
4179 : bool force_single_lane)
4180 : {
4181 : /* If there's no budget left bail out early. */
4182 2133951 : if (*limit == 0)
4183 : {
4184 25220 : scalar_stmts.release ();
4185 25220 : return false;
4186 : }
4187 :
4188 2108731 : if (kind == slp_inst_kind_ctor)
4189 : {
4190 14312 : if (dump_enabled_p ())
4191 86 : dump_printf_loc (MSG_NOTE, vect_location,
4192 : "Analyzing vectorizable constructor: %G\n",
4193 43 : root_stmt_infos[0]->stmt);
4194 : }
4195 2094419 : else if (kind == slp_inst_kind_gcond)
4196 : {
4197 290077 : if (dump_enabled_p ())
4198 5820 : dump_printf_loc (MSG_NOTE, vect_location,
4199 : "Analyzing vectorizable control flow: %G",
4200 2910 : root_stmt_infos[0]->stmt);
4201 : }
4202 1804342 : else if (kind == slp_inst_kind_bb_reduc)
4203 : {
4204 1417123 : if (dump_enabled_p ())
4205 6938 : dump_printf_loc (MSG_NOTE, vect_location,
4206 : "Analyzing vectorizable BB reduction: %G",
4207 3469 : root_stmt_infos[0]->stmt);
4208 : }
4209 :
4210 2108731 : if (dump_enabled_p ())
4211 : {
4212 26186 : dump_printf_loc (MSG_NOTE, vect_location,
4213 : "Starting SLP discovery for\n");
4214 56404 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4215 60436 : dump_printf_loc (MSG_NOTE, vect_location,
4216 30218 : " %G", scalar_stmts[i]->stmt);
4217 : }
4218 :
4219 : /* Build the tree for the SLP instance. */
4220 2108731 : unsigned int group_size = scalar_stmts.length ();
4221 2108731 : bool *matches = XALLOCAVEC (bool, group_size);
4222 2108731 : poly_uint64 max_nunits = 1;
4223 2108731 : unsigned tree_size = 0;
4224 :
4225 2108731 : slp_tree node = NULL;
4226 2108731 : if (group_size > 1 && force_single_lane)
4227 : {
4228 0 : matches[0] = true;
4229 0 : matches[1] = false;
4230 : }
4231 : else
4232 2108731 : node = vect_build_slp_tree (vinfo, scalar_stmts,
4233 : &max_nunits, matches, limit,
4234 : &tree_size, bst_map);
4235 2108731 : if (node != NULL)
4236 : {
4237 : /* Calculate the unrolling factor based on the smallest type. */
4238 817795 : poly_uint64 unrolling_factor
4239 817795 : = calculate_unrolling_factor (max_nunits, group_size);
4240 :
4241 817795 : if (maybe_ne (unrolling_factor, 1U)
4242 817795 : && is_a <bb_vec_info> (vinfo))
4243 : {
4244 0 : unsigned HOST_WIDE_INT const_max_nunits;
4245 0 : if (!max_nunits.is_constant (&const_max_nunits)
4246 0 : || const_max_nunits > group_size)
4247 : {
4248 0 : if (dump_enabled_p ())
4249 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4250 : "Build SLP failed: store group "
4251 : "size not a multiple of the vector size "
4252 : "in basic block SLP\n");
4253 0 : vect_free_slp_tree (node);
4254 0 : return false;
4255 : }
4256 : /* Fatal mismatch. */
4257 0 : if (dump_enabled_p ())
4258 0 : dump_printf_loc (MSG_NOTE, vect_location,
4259 : "SLP discovery succeeded but node needs "
4260 : "splitting\n");
4261 0 : memset (matches, true, group_size);
4262 0 : matches[group_size / const_max_nunits * const_max_nunits] = false;
4263 0 : vect_free_slp_tree (node);
4264 : }
4265 : else
4266 : {
4267 : /* Create a new SLP instance. */
4268 817795 : slp_instance new_instance = XNEW (class _slp_instance);
4269 817795 : SLP_INSTANCE_TREE (new_instance) = node;
4270 817795 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4271 817795 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4272 817795 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4273 817795 : SLP_INSTANCE_KIND (new_instance) = kind;
4274 817795 : new_instance->reduc_phis = NULL;
4275 817795 : new_instance->cost_vec = vNULL;
4276 817795 : new_instance->subgraph_entries = vNULL;
4277 :
4278 817795 : if (dump_enabled_p ())
4279 23077 : dump_printf_loc (MSG_NOTE, vect_location,
4280 : "SLP size %u vs. limit %u.\n",
4281 : tree_size, max_tree_size);
4282 :
4283 817795 : vinfo->slp_instances.safe_push (new_instance);
4284 :
4285 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4286 : the number of SLP lanes of the root in a few places.
4287 : Verify that assumption holds. */
4288 817795 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4289 : == group_size);
4290 :
4291 817795 : if (dump_enabled_p ())
4292 : {
4293 23077 : if (kind == slp_inst_kind_reduc_group)
4294 1470 : dump_printf_loc (MSG_NOTE, vect_location,
4295 : "SLP discovery of size %d reduction group "
4296 : "succeeded\n", group_size);
4297 23077 : dump_printf_loc (MSG_NOTE, vect_location,
4298 : "Final SLP tree for instance %p:\n",
4299 : (void *) new_instance);
4300 23077 : vect_print_slp_graph (MSG_NOTE, vect_location,
4301 : SLP_INSTANCE_TREE (new_instance));
4302 : }
4303 :
4304 817795 : return true;
4305 : }
4306 : }
4307 : /* Failed to SLP. */
4308 :
4309 : /* While we arrive here even with slp_inst_kind_store we should only
4310 : for group_size == 1. The code to split store groups is only in
4311 : vect_analyze_slp_instance now. */
4312 1290936 : gcc_assert (kind != slp_inst_kind_store || group_size == 1);
4313 :
4314 : /* For BB vectorization we get failures only in case of the need of
4315 : unrolling, as otherwise we'll simply get operands built from scalars.
4316 : Iff there is any mismatches in the toplevel stmts those will prevail,
4317 : otherwise we get the non-power-of-two tail of the lanes failed.
4318 : For BB reductions we mainly want to catch the first case so we pick
4319 : a more useful subset of lanes to reduce. */
4320 1290936 : if (kind == slp_inst_kind_bb_reduc && matches[0])
4321 : {
4322 : unsigned n_matching = 0;
4323 2184803 : for (unsigned i = 0; i < group_size; ++i)
4324 1545049 : if (matches[i])
4325 741841 : n_matching++;
4326 639754 : vec<stmt_vec_info> scalar_stmts2 = vNULL;
4327 : /* Try matched parts and put the rest to remain. */
4328 639754 : if (n_matching >= 2 && n_matching >= group_size / 2)
4329 : {
4330 : /* As we know the matches[] stmts match up, recursing for
4331 : non-power-of-two sizes will just force-fail the tail
4332 : for us at hopefully optimal vector size and succesfully
4333 : finish discovery. */
4334 47182 : scalar_stmts2.create (n_matching);
4335 244829 : for (unsigned i = 0; i < group_size; ++i)
4336 197647 : if (matches[i])
4337 141569 : scalar_stmts2.quick_push (scalar_stmts[i]);
4338 : else
4339 56078 : remain.safe_push
4340 59761 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4341 : }
4342 : /* Try the non-matching part. */
4343 592572 : else if (group_size - n_matching >= 2)
4344 : {
4345 : /* We do not know whether the !matches[] part matches, so avoid
4346 : cutting to a multiple of the vector size too early. We should
4347 : make progress by means of remain only growing and most of the
4348 : time prefering the matching[] part. */
4349 29535 : scalar_stmts2.create (scalar_stmts.length () - n_matching);
4350 250863 : for (unsigned i = 0; i < group_size; ++i)
4351 221328 : if (!matches[i])
4352 184093 : scalar_stmts2.quick_push (scalar_stmts[i]);
4353 : else
4354 37235 : remain.safe_push
4355 38235 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4356 : }
4357 639754 : if (scalar_stmts2.exists ())
4358 : {
4359 76717 : if (dump_enabled_p ())
4360 202 : dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d "
4361 : "non-matching lanes to scalar remains\n",
4362 101 : scalar_stmts.length () - scalar_stmts2.length ());
4363 76717 : scalar_stmts.release ();
4364 76717 : return vect_build_slp_instance (vinfo, kind, scalar_stmts2,
4365 : root_stmt_infos, remain,
4366 : max_tree_size, limit, bst_map,
4367 76717 : force_single_lane);
4368 : }
4369 : }
4370 :
4371 : /* Free the allocated memory. */
4372 1214219 : scalar_stmts.release ();
4373 :
4374 : /* Failed to SLP. */
4375 1214219 : if (dump_enabled_p ())
4376 3008 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4377 : return false;
4378 : }
4379 :
4380 : /* Analyze an SLP instance starting from a the start of a reduction chain.
4381 : Call vect_build_slp_tree to build a tree of packed stmts if possible.
4382 : Return FALSE if SLP build fails. */
4383 :
4384 : static bool
4385 73324 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
4386 : scalar_stmts_to_slp_tree_map_t *bst_map,
4387 : stmt_vec_info scalar_stmt,
4388 : unsigned max_tree_size, unsigned *limit)
4389 : {
4390 73324 : vec<stmt_vec_info> scalar_stmts = vNULL;
4391 :
4392 73324 : bool fail = false;
4393 : /* ??? We could leave operation code checking to SLP discovery. */
4394 73324 : code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
4395 : (vect_orig_stmt (scalar_stmt)));
4396 73324 : bool first = true;
4397 73324 : stmt_vec_info next_stmt = scalar_stmt;
4398 83314 : do
4399 : {
4400 83314 : stmt_vec_info stmt = next_stmt;
4401 83314 : gimple_match_op op, orig_op;
4402 83314 : if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
4403 0 : gcc_unreachable ();
4404 166628 : tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
4405 83314 : STMT_VINFO_REDUC_IDX (stmt));
4406 83314 : next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
4407 83314 : gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
4408 : || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
4409 89222 : if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)),
4410 : &orig_op))
4411 0 : gcc_unreachable ();
4412 83314 : if (CONVERT_EXPR_CODE_P (op.code)
4413 4989 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
4414 88291 : && (first
4415 2496 : || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
4416 : ;
4417 78397 : else if (code != orig_op.code)
4418 : {
4419 2726 : fail = true;
4420 2726 : break;
4421 : }
4422 : else
4423 75671 : scalar_stmts.safe_push (stmt);
4424 80588 : first = false;
4425 : }
4426 80588 : while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
4427 73324 : if (fail)
4428 2726 : return false;
4429 :
4430 : /* Remember a stmt with the actual reduction operation. */
4431 70598 : stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
4432 :
4433 : /* When the SSA def chain through reduc-idx does not form a natural
4434 : reduction chain try to linearize an associative operation manually. */
4435 70598 : if (scalar_stmts.length () == 1
4436 67975 : && code.is_tree_code ()
4437 61904 : && associative_tree_code ((tree_code)code)
4438 : /* We may not associate if a fold-left reduction is required. */
4439 131084 : && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
4440 : (reduc_scalar_stmt->stmt)),
4441 : code))
4442 : {
4443 57229 : auto_vec<chain_op_t> chain;
4444 57229 : auto_vec<std::pair<tree_code, gimple *> > worklist;
4445 57229 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
4446 57229 : if (is_a <gassign *> (scalar_stmts[0]->stmt)
4447 : /* We cannot linearize an operation that vect_slp_linearize_chain
4448 : would not put on its worklist. */
4449 57229 : && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
4450 : {
4451 56576 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4452 56576 : scalar_stmts[0]->stmt, op_stmt,
4453 : other_op_stmt,
4454 : NULL);
4455 :
4456 56576 : scalar_stmts.truncate (0);
4457 56576 : stmt_vec_info tail = NULL;
4458 283065 : for (auto el : chain)
4459 : {
4460 113700 : if (el.dt == vect_external_def
4461 113700 : || el.dt == vect_constant_def
4462 113700 : || el.code != (tree_code) code)
4463 : {
4464 363 : scalar_stmts.release ();
4465 363 : return false;
4466 : }
4467 113337 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4468 113337 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4469 111053 : || STMT_VINFO_REDUC_DEF (stmt))
4470 : {
4471 56446 : gcc_assert (tail == NULL);
4472 56446 : tail = stmt;
4473 56446 : continue;
4474 : }
4475 56891 : scalar_stmts.safe_push (stmt);
4476 : }
4477 56213 : gcc_assert (tail);
4478 : }
4479 :
4480 : /* When this linearization didn't produce a chain see if stripping
4481 : a wrapping sign conversion produces one. */
4482 56866 : if (scalar_stmts.length () == 1
4483 56866 : && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
4484 : || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
4485 : {
4486 55079 : gimple *stmt = scalar_stmts[0]->stmt;
4487 55079 : if (!is_gimple_assign (stmt)
4488 53912 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
4489 4724 : || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
4490 59803 : || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
4491 4724 : TREE_TYPE (gimple_assign_rhs1 (stmt))))
4492 : {
4493 53300 : scalar_stmts.release ();
4494 53300 : return false;
4495 : }
4496 1779 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
4497 1779 : if (!is_gimple_assign (stmt)
4498 1779 : || gimple_assign_rhs_code (stmt) != (tree_code)code)
4499 : {
4500 1760 : scalar_stmts.release ();
4501 1760 : return false;
4502 : }
4503 19 : chain.truncate (0);
4504 19 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4505 : stmt, op_stmt, other_op_stmt, NULL);
4506 :
4507 19 : scalar_stmts.truncate (0);
4508 19 : stmt_vec_info tail = NULL;
4509 93 : for (auto el : chain)
4510 : {
4511 44 : if (el.dt == vect_external_def
4512 44 : || el.dt == vect_constant_def
4513 44 : || el.code != (tree_code) code)
4514 : {
4515 8 : scalar_stmts.release ();
4516 8 : return false;
4517 : }
4518 36 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4519 36 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4520 36 : || STMT_VINFO_REDUC_DEF (stmt))
4521 : {
4522 0 : gcc_assert (tail == NULL);
4523 0 : tail = stmt;
4524 0 : continue;
4525 : }
4526 36 : scalar_stmts.safe_push (stmt);
4527 : }
4528 : /* Unlike the above this does not include the reduction SSA
4529 : cycle. */
4530 11 : gcc_assert (!tail);
4531 : }
4532 :
4533 1798 : if (scalar_stmts.length () < 2)
4534 : {
4535 1673 : scalar_stmts.release ();
4536 1673 : return false;
4537 : }
4538 :
4539 125 : if (dump_enabled_p ())
4540 : {
4541 34 : dump_printf_loc (MSG_NOTE, vect_location,
4542 : "Starting SLP discovery of reduction chain for\n");
4543 140 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4544 212 : dump_printf_loc (MSG_NOTE, vect_location,
4545 106 : " %G", scalar_stmts[i]->stmt);
4546 : }
4547 :
4548 125 : unsigned int group_size = scalar_stmts.length ();
4549 125 : bool *matches = XALLOCAVEC (bool, group_size);
4550 125 : poly_uint64 max_nunits = 1;
4551 125 : unsigned tree_size = 0;
4552 125 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4553 : &max_nunits, matches, limit,
4554 125 : &tree_size, bst_map);
4555 125 : if (!node)
4556 : {
4557 47 : scalar_stmts.release ();
4558 47 : return false;
4559 : }
4560 :
4561 78 : unsigned cycle_id = vinfo->reduc_infos.length ();
4562 78 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
4563 78 : vinfo->reduc_infos.safe_push (reduc_info);
4564 78 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
4565 78 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
4566 78 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
4567 78 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
4568 78 : reduc_info->is_reduc_chain = true;
4569 :
4570 : /* Build the node for the PHI and possibly the conversions. */
4571 78 : slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
4572 78 : SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
4573 78 : phis->cycle_info.id = cycle_id;
4574 78 : SLP_TREE_LANES (phis) = group_size;
4575 78 : if (reduc_scalar_stmt == scalar_stmt)
4576 74 : SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
4577 : else
4578 4 : SLP_TREE_VECTYPE (phis)
4579 4 : = signed_or_unsigned_type_for (TYPE_UNSIGNED
4580 : (TREE_TYPE (gimple_get_lhs
4581 : (scalar_stmt->stmt))),
4582 : SLP_TREE_VECTYPE (node));
4583 : /* ??? vect_cse_slp_nodes cannot cope with cycles without any
4584 : SLP_TREE_SCALAR_STMTS. */
4585 78 : SLP_TREE_SCALAR_STMTS (phis).create (group_size);
4586 393 : for (unsigned i = 0; i < group_size; ++i)
4587 315 : SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
4588 :
4589 78 : slp_tree op_input = phis;
4590 78 : if (reduc_scalar_stmt != scalar_stmt)
4591 : {
4592 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4593 4 : SLP_TREE_REPRESENTATIVE (conv)
4594 4 : = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
4595 4 : STMT_VINFO_REDUC_IDX
4596 : (reduc_scalar_stmt)));
4597 4 : SLP_TREE_CHILDREN (conv).quick_push (phis);
4598 4 : conv->cycle_info.id = cycle_id;
4599 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4600 4 : SLP_TREE_LANES (conv) = group_size;
4601 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
4602 4 : SLP_TREE_SCALAR_STMTS (conv) = vNULL;
4603 4 : op_input = conv;
4604 : }
4605 :
4606 78 : slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
4607 78 : SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
4608 78 : SLP_TREE_CHILDREN (reduc).quick_push (op_input);
4609 78 : SLP_TREE_CHILDREN (reduc).quick_push (node);
4610 78 : reduc->cycle_info.id = cycle_id;
4611 78 : SLP_TREE_REDUC_IDX (reduc) = 0;
4612 78 : SLP_TREE_LANES (reduc) = group_size;
4613 78 : SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
4614 : /* ??? For the reduction epilogue we need a live lane. */
4615 78 : SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
4616 78 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
4617 315 : for (unsigned i = 1; i < group_size; ++i)
4618 237 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
4619 :
4620 78 : if (reduc_scalar_stmt != scalar_stmt)
4621 : {
4622 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4623 4 : SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
4624 4 : SLP_TREE_CHILDREN (conv).quick_push (reduc);
4625 4 : conv->cycle_info.id = cycle_id;
4626 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4627 4 : SLP_TREE_LANES (conv) = group_size;
4628 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
4629 : /* ??? For the reduction epilogue we need a live lane. */
4630 4 : SLP_TREE_SCALAR_STMTS (conv).create (group_size);
4631 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
4632 8 : for (unsigned i = 1; i < group_size; ++i)
4633 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
4634 4 : reduc = conv;
4635 : }
4636 :
4637 78 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
4638 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4639 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4640 78 : SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
4641 78 : SLP_TREE_REF_COUNT (reduc)++;
4642 :
4643 : /* Create a new SLP instance. */
4644 78 : slp_instance new_instance = XNEW (class _slp_instance);
4645 78 : SLP_INSTANCE_TREE (new_instance) = reduc;
4646 78 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4647 78 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4648 78 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4649 78 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4650 78 : new_instance->reduc_phis = NULL;
4651 78 : new_instance->cost_vec = vNULL;
4652 78 : new_instance->subgraph_entries = vNULL;
4653 :
4654 78 : vinfo->slp_instances.safe_push (new_instance);
4655 :
4656 78 : if (dump_enabled_p ())
4657 : {
4658 24 : dump_printf_loc (MSG_NOTE, vect_location,
4659 : "Final SLP tree for instance %p:\n",
4660 : (void *) new_instance);
4661 24 : vect_print_slp_graph (MSG_NOTE, vect_location,
4662 : SLP_INSTANCE_TREE (new_instance));
4663 : }
4664 :
4665 78 : return true;
4666 57229 : }
4667 :
4668 13369 : if (scalar_stmts.length () <= 1)
4669 : {
4670 10746 : scalar_stmts.release ();
4671 10746 : return false;
4672 : }
4673 :
4674 2623 : scalar_stmts.reverse ();
4675 2623 : stmt_vec_info reduc_phi_info = next_stmt;
4676 :
4677 : /* Build the tree for the SLP instance. */
4678 2623 : vec<stmt_vec_info> root_stmt_infos = vNULL;
4679 2623 : vec<tree> remain = vNULL;
4680 :
4681 2623 : if (dump_enabled_p ())
4682 : {
4683 196 : dump_printf_loc (MSG_NOTE, vect_location,
4684 : "Starting SLP discovery of reduction chain for\n");
4685 1038 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4686 1684 : dump_printf_loc (MSG_NOTE, vect_location,
4687 842 : " %G", scalar_stmts[i]->stmt);
4688 : }
4689 :
4690 : /* Build the tree for the SLP instance. */
4691 2623 : unsigned int group_size = scalar_stmts.length ();
4692 2623 : bool *matches = XALLOCAVEC (bool, group_size);
4693 2623 : poly_uint64 max_nunits = 1;
4694 2623 : unsigned tree_size = 0;
4695 :
4696 : /* ??? We need this only for SLP discovery. */
4697 10237 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4698 7614 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
4699 :
4700 2623 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4701 : &max_nunits, matches, limit,
4702 2623 : &tree_size, bst_map);
4703 :
4704 10237 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4705 7614 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
4706 :
4707 2623 : if (node != NULL)
4708 : {
4709 : /* Create a new SLP instance. */
4710 2339 : slp_instance new_instance = XNEW (class _slp_instance);
4711 2339 : SLP_INSTANCE_TREE (new_instance) = node;
4712 2339 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4713 2339 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4714 2339 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4715 2339 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4716 2339 : new_instance->reduc_phis = NULL;
4717 2339 : new_instance->cost_vec = vNULL;
4718 2339 : new_instance->subgraph_entries = vNULL;
4719 :
4720 2339 : vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
4721 2339 : reduc_info->is_reduc_chain = true;
4722 :
4723 2339 : if (dump_enabled_p ())
4724 147 : dump_printf_loc (MSG_NOTE, vect_location,
4725 : "SLP size %u vs. limit %u.\n",
4726 : tree_size, max_tree_size);
4727 :
4728 : /* Fixup SLP reduction chains. If this is a reduction chain with
4729 : a conversion in front amend the SLP tree with a node for that. */
4730 2339 : gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
4731 2339 : if (is_gimple_assign (scalar_def)
4732 2339 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
4733 : {
4734 43 : stmt_vec_info conv_info = vect_stmt_to_vectorize
4735 43 : (STMT_VINFO_REDUC_DEF (reduc_phi_info));
4736 43 : scalar_stmts = vNULL;
4737 43 : scalar_stmts.create (group_size);
4738 135 : for (unsigned i = 0; i < group_size; ++i)
4739 92 : scalar_stmts.quick_push (conv_info);
4740 43 : slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
4741 43 : SLP_TREE_VECTYPE (conv)
4742 43 : = get_vectype_for_scalar_type (vinfo,
4743 43 : TREE_TYPE
4744 : (gimple_assign_lhs (scalar_def)),
4745 : group_size);
4746 43 : SLP_TREE_REDUC_IDX (conv) = 0;
4747 43 : conv->cycle_info.id = node->cycle_info.id;
4748 43 : SLP_TREE_CHILDREN (conv).quick_push (node);
4749 43 : SLP_INSTANCE_TREE (new_instance) = conv;
4750 : }
4751 : /* Fill the backedge child of the PHI SLP node. The
4752 : general matching code cannot find it because the
4753 : scalar code does not reflect how we vectorize the
4754 : reduction. */
4755 2339 : use_operand_p use_p;
4756 2339 : imm_use_iterator imm_iter;
4757 2339 : class loop *loop = LOOP_VINFO_LOOP (vinfo);
4758 11235 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
4759 : gimple_get_lhs (scalar_def))
4760 : /* There are exactly two non-debug uses, the reduction
4761 : PHI and the loop-closed PHI node. */
4762 6557 : if (!is_gimple_debug (USE_STMT (use_p))
4763 6557 : && gimple_bb (USE_STMT (use_p)) == loop->header)
4764 : {
4765 2339 : auto_vec<stmt_vec_info, 64> phis (group_size);
4766 2339 : stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
4767 9223 : for (unsigned i = 0; i < group_size; ++i)
4768 6884 : phis.quick_push (phi_info);
4769 2339 : slp_tree *phi_node = bst_map->get (phis);
4770 2339 : unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
4771 4678 : SLP_TREE_CHILDREN (*phi_node)[dest_idx]
4772 2339 : = SLP_INSTANCE_TREE (new_instance);
4773 2339 : SLP_INSTANCE_TREE (new_instance)->refcnt++;
4774 2339 : }
4775 :
4776 2339 : vinfo->slp_instances.safe_push (new_instance);
4777 :
4778 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4779 : the number of SLP lanes of the root in a few places.
4780 : Verify that assumption holds. */
4781 2339 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4782 : == group_size);
4783 :
4784 2339 : if (dump_enabled_p ())
4785 : {
4786 147 : dump_printf_loc (MSG_NOTE, vect_location,
4787 : "Final SLP tree for instance %p:\n",
4788 : (void *) new_instance);
4789 147 : vect_print_slp_graph (MSG_NOTE, vect_location,
4790 : SLP_INSTANCE_TREE (new_instance));
4791 : }
4792 :
4793 2339 : return true;
4794 : }
4795 :
4796 : /* Failed to SLP. */
4797 284 : scalar_stmts.release ();
4798 284 : if (dump_enabled_p ())
4799 49 : dump_printf_loc (MSG_NOTE, vect_location,
4800 : "SLP discovery of reduction chain failed\n");
4801 : return false;
4802 : }
4803 :
4804 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4805 : of KIND. Return true if successful. */
4806 :
4807 : static bool
4808 100169 : vect_analyze_slp_reduction (loop_vec_info vinfo,
4809 : stmt_vec_info scalar_stmt,
4810 : unsigned max_tree_size, unsigned *limit,
4811 : scalar_stmts_to_slp_tree_map_t *bst_map,
4812 : bool force_single_lane)
4813 : {
4814 100169 : slp_instance_kind kind = slp_inst_kind_reduc_group;
4815 :
4816 : /* Try to gather a reduction chain. Only attempt if there's budget left
4817 : since chain analysis may build multi-lane trees that consume limit. */
4818 100169 : if (! force_single_lane
4819 73609 : && *limit != 0
4820 73609 : && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
4821 173493 : && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
4822 : max_tree_size, limit))
4823 : return true;
4824 :
4825 97752 : vec<stmt_vec_info> scalar_stmts;
4826 97752 : scalar_stmts.create (1);
4827 97752 : scalar_stmts.quick_push (scalar_stmt);
4828 :
4829 97752 : if (dump_enabled_p ())
4830 : {
4831 3908 : dump_printf_loc (MSG_NOTE, vect_location,
4832 : "Starting SLP discovery for\n");
4833 7816 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4834 7816 : dump_printf_loc (MSG_NOTE, vect_location,
4835 3908 : " %G", scalar_stmts[i]->stmt);
4836 : }
4837 :
4838 : /* Build the tree for the SLP instance. */
4839 97752 : unsigned int group_size = scalar_stmts.length ();
4840 97752 : bool *matches = XALLOCAVEC (bool, group_size);
4841 97752 : poly_uint64 max_nunits = 1;
4842 97752 : unsigned tree_size = 0;
4843 :
4844 97752 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4845 : &max_nunits, matches, limit,
4846 : &tree_size, bst_map);
4847 97752 : if (node != NULL)
4848 : {
4849 : /* Create a new SLP instance. */
4850 95852 : slp_instance new_instance = XNEW (class _slp_instance);
4851 95852 : SLP_INSTANCE_TREE (new_instance) = node;
4852 95852 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4853 95852 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4854 95852 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4855 95852 : SLP_INSTANCE_KIND (new_instance) = kind;
4856 95852 : new_instance->reduc_phis = NULL;
4857 95852 : new_instance->cost_vec = vNULL;
4858 95852 : new_instance->subgraph_entries = vNULL;
4859 :
4860 95852 : if (dump_enabled_p ())
4861 3827 : dump_printf_loc (MSG_NOTE, vect_location,
4862 : "SLP size %u vs. limit %u.\n",
4863 : tree_size, max_tree_size);
4864 :
4865 95852 : vinfo->slp_instances.safe_push (new_instance);
4866 :
4867 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4868 : the number of SLP lanes of the root in a few places.
4869 : Verify that assumption holds. */
4870 95852 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4871 : == group_size);
4872 :
4873 95852 : if (dump_enabled_p ())
4874 : {
4875 3827 : dump_printf_loc (MSG_NOTE, vect_location,
4876 : "Final SLP tree for instance %p:\n",
4877 : (void *) new_instance);
4878 3827 : vect_print_slp_graph (MSG_NOTE, vect_location,
4879 : SLP_INSTANCE_TREE (new_instance));
4880 : }
4881 :
4882 95852 : return true;
4883 : }
4884 : /* Failed to SLP. */
4885 :
4886 : /* Free the allocated memory. */
4887 1900 : scalar_stmts.release ();
4888 :
4889 : /* Failed to SLP. */
4890 1900 : if (dump_enabled_p ())
4891 81 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4892 : return false;
4893 : }
4894 :
4895 : /* Analyze a single SLP reduction group. If successful add a SLP instance
4896 : for it and return true, otherwise return false and have *MATCHES
4897 : populated. */
4898 :
4899 : static bool
4900 24150 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
4901 : vec<stmt_vec_info> scalar_stmts,
4902 : scalar_stmts_to_slp_tree_map_t *bst_map,
4903 : unsigned max_tree_size, unsigned *limit,
4904 : bool *matches)
4905 : {
4906 : /* Try to form a reduction group. Size-1 groups are not suitable
4907 : for SLP reduction and should fall back to single-lane reduction. */
4908 45536 : unsigned int group_size = scalar_stmts.length ();
4909 24150 : if (group_size <= 1)
4910 : return false;
4911 17471 : if (!matches)
4912 4550 : matches = XALLOCAVEC (bool, group_size);
4913 17471 : poly_uint64 max_nunits = 1;
4914 17471 : unsigned tree_size = 0;
4915 17471 : slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts,
4916 : &max_nunits, matches, limit,
4917 : &tree_size, bst_map);
4918 17471 : if (!node)
4919 : return false;
4920 :
4921 : /* Create a new SLP instance. */
4922 2764 : slp_instance new_instance = XNEW (class _slp_instance);
4923 2764 : SLP_INSTANCE_TREE (new_instance) = node;
4924 2764 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4925 2764 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4926 2764 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4927 2764 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
4928 2764 : new_instance->reduc_phis = NULL;
4929 2764 : new_instance->cost_vec = vNULL;
4930 2764 : new_instance->subgraph_entries = vNULL;
4931 :
4932 2764 : if (dump_enabled_p ())
4933 211 : dump_printf_loc (MSG_NOTE, vect_location,
4934 : "SLP size %u vs. limit %u.\n",
4935 : tree_size, max_tree_size);
4936 :
4937 2764 : loop_vinfo->slp_instances.safe_push (new_instance);
4938 :
4939 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4940 : the number of SLP lanes of the root in a few places.
4941 : Verify that assumption holds. */
4942 2764 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4943 : == group_size);
4944 :
4945 2764 : if (dump_enabled_p ())
4946 : {
4947 211 : dump_printf_loc (MSG_NOTE, vect_location,
4948 : "SLP discovery of size %d reduction group "
4949 : "succeeded\n", group_size);
4950 211 : dump_printf_loc (MSG_NOTE, vect_location,
4951 : "Final SLP tree for instance %p:\n",
4952 : (void *) new_instance);
4953 211 : vect_print_slp_graph (MSG_NOTE, vect_location,
4954 : SLP_INSTANCE_TREE (new_instance));
4955 : }
4956 :
4957 : return true;
4958 : }
4959 :
4960 : /* Analyze reductions in LOOP_VINFO and populate SLP instances
4961 : accordingly. Returns false if something fails. */
4962 :
4963 : static bool
4964 511292 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
4965 : unsigned max_tree_size, unsigned *limit,
4966 : scalar_stmts_to_slp_tree_map_t *bst_map,
4967 : bool force_single_lane)
4968 : {
4969 582224 : if (loop_vinfo->reductions.is_empty ())
4970 : return true;
4971 :
4972 : /* Collect reduction statements we can combine into
4973 : a SLP reduction. */
4974 74389 : vec<stmt_vec_info> scalar_stmts;
4975 74389 : scalar_stmts.create (loop_vinfo->reductions.length ());
4976 329726 : for (auto next_info : loop_vinfo->reductions)
4977 : {
4978 106559 : next_info = vect_stmt_to_vectorize (next_info);
4979 106559 : if ((STMT_VINFO_RELEVANT_P (next_info)
4980 14 : || STMT_VINFO_LIVE_P (next_info))
4981 : /* ??? Make sure we didn't skip a conversion around a
4982 : reduction path. In that case we'd have to reverse
4983 : engineer that conversion stmt following the chain using
4984 : reduc_idx and from the PHI using reduc_def. */
4985 106545 : && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
4986 106545 : || (STMT_VINFO_DEF_TYPE (next_info)
4987 : == vect_double_reduction_def)))
4988 : {
4989 : /* Do not discover SLP reductions combining lane-reducing
4990 : ops, that will fail later. */
4991 106545 : if (!force_single_lane
4992 106545 : && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
4993 79288 : scalar_stmts.quick_push (next_info);
4994 : /* Do SLP discovery for single-lane reductions. */
4995 27257 : else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
4996 : max_tree_size, limit,
4997 : bst_map,
4998 : force_single_lane))
4999 : {
5000 0 : scalar_stmts.release ();
5001 0 : return false;
5002 : }
5003 : }
5004 : }
5005 :
5006 74389 : if (scalar_stmts.length () > 1)
5007 : {
5008 : /* Try to form a reduction group. */
5009 4644 : unsigned int group_size = scalar_stmts.length ();
5010 4644 : bool *matches = XALLOCAVEC (bool, group_size);
5011 4644 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
5012 : max_tree_size, limit, matches))
5013 1579 : return true;
5014 :
5015 : /* When analysis as a single SLP reduction group failed try to
5016 : form sub-groups by collecting matching lanes. Do not recurse
5017 : that on failure (to limit compile-time costs), but recurse
5018 : for the initial non-matching parts. Everything not covered
5019 : by a sub-group gets single-reduction treatment. */
5020 3517 : vec<stmt_vec_info> cands = vNULL;
5021 11364 : while (matches[0])
5022 : {
5023 11229 : cands.truncate (0);
5024 11229 : cands.reserve (group_size, true);
5025 88339 : for (unsigned i = 0; i < group_size; ++i)
5026 77110 : if (matches[i])
5027 19562 : cands.quick_push (scalar_stmts[i]);
5028 :
5029 : /* Try to form a reduction group. */
5030 11229 : if (vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
5031 : max_tree_size, limit, NULL))
5032 1207 : cands = vNULL;
5033 : else
5034 : {
5035 : /* Do SLP discovery for single-lane reductions. */
5036 47164 : for (auto stmt_info : cands)
5037 17120 : if (! vect_analyze_slp_reduction (loop_vinfo,
5038 : vect_stmt_to_vectorize
5039 : (stmt_info),
5040 : max_tree_size, limit,
5041 : bst_map, force_single_lane))
5042 : {
5043 22 : scalar_stmts.release ();
5044 22 : cands.release ();
5045 22 : return false;
5046 : }
5047 : }
5048 : /* Remove the handled stmts from scalar_stmts and try again,
5049 : possibly repeating the above with updated matches[]. */
5050 : unsigned j = 0;
5051 88255 : for (unsigned i = 0; i < group_size; ++i)
5052 77048 : if (!matches[i])
5053 : {
5054 57521 : scalar_stmts[j] = scalar_stmts[i];
5055 57521 : ++j;
5056 : }
5057 11207 : scalar_stmts.truncate (j);
5058 11207 : group_size = scalar_stmts.length ();
5059 11207 : if (group_size <= 1)
5060 : break;
5061 8277 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts,
5062 : bst_map, max_tree_size, limit,
5063 : matches))
5064 : return true;
5065 : }
5066 : }
5067 : /* Do SLP discovery for single-lane reductions. */
5068 272344 : for (auto stmt_info : scalar_stmts)
5069 55792 : if (! vect_analyze_slp_reduction (loop_vinfo,
5070 : vect_stmt_to_vectorize (stmt_info),
5071 : max_tree_size, limit,
5072 : bst_map, force_single_lane))
5073 : {
5074 1878 : scalar_stmts.release ();
5075 1878 : return false;
5076 : }
5077 :
5078 70932 : scalar_stmts.release ();
5079 70932 : return true;
5080 : }
5081 :
5082 : /* Analyze an SLP instance starting from a group of grouped stores. Call
5083 : vect_build_slp_tree to build a tree of packed stmts if possible.
5084 : Return FALSE if it's impossible to SLP any stmt in the group. */
5085 :
5086 : static bool
5087 1117599 : vect_analyze_slp_instance (vec_info *vinfo,
5088 : scalar_stmts_to_slp_tree_map_t *bst_map,
5089 : stmt_vec_info stmt_info,
5090 : slp_instance_kind kind,
5091 : unsigned max_tree_size, unsigned *limit,
5092 : bool force_single_lane)
5093 : {
5094 1117599 : vec<stmt_vec_info> scalar_stmts;
5095 :
5096 1117599 : if (is_a <bb_vec_info> (vinfo))
5097 1087438 : vect_location = stmt_info->stmt;
5098 :
5099 1117599 : gcc_assert (kind == slp_inst_kind_store);
5100 :
5101 : /* Collect the stores and store them in scalar_stmts. */
5102 1117599 : scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
5103 1117599 : stmt_vec_info next_info = stmt_info;
5104 5556294 : while (next_info)
5105 : {
5106 3321096 : scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
5107 3321096 : next_info = DR_GROUP_NEXT_ELEMENT (next_info);
5108 : }
5109 :
5110 1117599 : vec<stmt_vec_info> root_stmt_infos = vNULL;
5111 1117599 : vec<tree> remain = vNULL;
5112 :
5113 : /* Build the tree for the SLP instance. */
5114 :
5115 : /* If there's no budget left bail out early. */
5116 1117599 : if (*limit == 0)
5117 : return false;
5118 :
5119 1117579 : if (dump_enabled_p ())
5120 : {
5121 4141 : dump_printf_loc (MSG_NOTE, vect_location,
5122 : "Starting SLP discovery for\n");
5123 24251 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
5124 40220 : dump_printf_loc (MSG_NOTE, vect_location,
5125 20110 : " %G", scalar_stmts[i]->stmt);
5126 : }
5127 :
5128 : /* Build the tree for the SLP instance. */
5129 1117579 : unsigned int group_size = scalar_stmts.length ();
5130 1117579 : bool *matches = XALLOCAVEC (bool, group_size);
5131 1117579 : poly_uint64 max_nunits = 1;
5132 1117579 : unsigned tree_size = 0;
5133 1117579 : unsigned i;
5134 :
5135 1117579 : slp_tree node = NULL;
5136 1117579 : if (group_size > 1 && force_single_lane)
5137 : {
5138 1771 : matches[0] = true;
5139 1771 : matches[1] = false;
5140 : }
5141 : else
5142 1115808 : node = vect_build_slp_tree (vinfo, scalar_stmts,
5143 : &max_nunits, matches, limit,
5144 : &tree_size, bst_map);
5145 1117579 : if (node != NULL)
5146 : {
5147 : /* Calculate the unrolling factor based on the smallest type. */
5148 693101 : poly_uint64 unrolling_factor
5149 693101 : = calculate_unrolling_factor (max_nunits, group_size);
5150 :
5151 693101 : if (maybe_ne (unrolling_factor, 1U)
5152 693101 : && is_a <bb_vec_info> (vinfo))
5153 : {
5154 0 : unsigned HOST_WIDE_INT const_max_nunits;
5155 0 : if (!max_nunits.is_constant (&const_max_nunits)
5156 0 : || const_max_nunits > group_size)
5157 : {
5158 0 : if (dump_enabled_p ())
5159 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5160 : "Build SLP failed: store group "
5161 : "size not a multiple of the vector size "
5162 : "in basic block SLP\n");
5163 0 : vect_free_slp_tree (node);
5164 0 : return false;
5165 : }
5166 : /* Fatal mismatch. */
5167 0 : if (dump_enabled_p ())
5168 0 : dump_printf_loc (MSG_NOTE, vect_location,
5169 : "SLP discovery succeeded but node needs "
5170 : "splitting\n");
5171 0 : memset (matches, true, group_size);
5172 0 : matches[group_size / const_max_nunits * const_max_nunits] = false;
5173 0 : vect_free_slp_tree (node);
5174 : }
5175 : else
5176 : {
5177 : /* Create a new SLP instance. */
5178 693101 : slp_instance new_instance = XNEW (class _slp_instance);
5179 693101 : SLP_INSTANCE_TREE (new_instance) = node;
5180 693101 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5181 693101 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5182 693101 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5183 693101 : SLP_INSTANCE_KIND (new_instance) = kind;
5184 693101 : new_instance->reduc_phis = NULL;
5185 693101 : new_instance->cost_vec = vNULL;
5186 693101 : new_instance->subgraph_entries = vNULL;
5187 :
5188 693101 : if (dump_enabled_p ())
5189 3142 : dump_printf_loc (MSG_NOTE, vect_location,
5190 : "SLP size %u vs. limit %u.\n",
5191 : tree_size, max_tree_size);
5192 :
5193 693101 : vinfo->slp_instances.safe_push (new_instance);
5194 :
5195 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5196 : the number of SLP lanes of the root in a few places.
5197 : Verify that assumption holds. */
5198 693101 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5199 : == group_size);
5200 :
5201 693101 : if (dump_enabled_p ())
5202 : {
5203 3142 : dump_printf_loc (MSG_NOTE, vect_location,
5204 : "Final SLP tree for instance %p:\n",
5205 : (void *) new_instance);
5206 3142 : vect_print_slp_graph (MSG_NOTE, vect_location,
5207 : SLP_INSTANCE_TREE (new_instance));
5208 : }
5209 :
5210 693101 : return true;
5211 : }
5212 : }
5213 : /* Failed to SLP. */
5214 :
5215 : /* Try to break the group up into pieces. */
5216 424478 : if (*limit > 0 && kind == slp_inst_kind_store)
5217 : {
5218 : /* ??? We could delay all the actual splitting of store-groups
5219 : until after SLP discovery of the original group completed.
5220 : Then we can recurse to vect_build_slp_instance directly. */
5221 1111127 : for (i = 0; i < group_size; i++)
5222 1111127 : if (!matches[i])
5223 : break;
5224 :
5225 : /* For basic block SLP, try to break the group up into multiples of
5226 : a vector size. */
5227 424477 : if (is_a <bb_vec_info> (vinfo)
5228 424477 : && (i > 1 && i < group_size))
5229 : {
5230 : /* Free the allocated memory. */
5231 159251 : scalar_stmts.release ();
5232 :
5233 159251 : tree scalar_type
5234 159251 : = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
5235 318502 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
5236 159251 : 1 << floor_log2 (i));
5237 159251 : unsigned HOST_WIDE_INT const_nunits;
5238 159251 : if (vectype
5239 159251 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
5240 : {
5241 : /* Split into two groups at the first vector boundary. */
5242 159251 : gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
5243 159251 : unsigned group1_size = i & ~(const_nunits - 1);
5244 :
5245 159251 : if (dump_enabled_p ())
5246 66 : dump_printf_loc (MSG_NOTE, vect_location,
5247 : "Splitting SLP group at stmt %u\n", i);
5248 159251 : stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
5249 : group1_size);
5250 159251 : bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
5251 : kind, max_tree_size,
5252 : limit, false);
5253 : /* Split the rest at the failure point and possibly
5254 : re-analyze the remaining matching part if it has
5255 : at least two lanes. */
5256 159251 : if (group1_size < i
5257 5673 : && (i + 1 < group_size
5258 3070 : || i - group1_size > 1))
5259 : {
5260 2631 : stmt_vec_info rest2 = rest;
5261 2631 : rest = vect_split_slp_store_group (rest, i - group1_size);
5262 2631 : if (i - group1_size > 1)
5263 57 : res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
5264 : kind, max_tree_size,
5265 : limit, false);
5266 : }
5267 : /* Re-analyze the non-matching tail if it has at least
5268 : two lanes. */
5269 159251 : if (i + 1 < group_size)
5270 22489 : res |= vect_analyze_slp_instance (vinfo, bst_map,
5271 : rest, kind, max_tree_size,
5272 : limit, false);
5273 159251 : return res;
5274 : }
5275 : }
5276 :
5277 : /* For loop vectorization split the RHS into arbitrary pieces of
5278 : size >= 1. */
5279 265226 : else if (is_a <loop_vec_info> (vinfo)
5280 265226 : && (group_size != 1 && i < group_size))
5281 : {
5282 8360 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
5283 28 : bool masked_p = call
5284 28 : && gimple_call_internal_p (call)
5285 28 : && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
5286 : /* There are targets that cannot do even/odd interleaving schemes
5287 : so they absolutely need to use load/store-lanes. For now
5288 : force single-lane SLP for them - they would be happy with
5289 : uniform power-of-two lanes (but depending on element size),
5290 : but even if we can use 'i' as indicator we would need to
5291 : backtrack when later lanes fail to discover with the same
5292 : granularity. We cannot turn any of strided or scatter store
5293 : into store-lanes. */
5294 : /* ??? If this is not in sync with what get_load_store_type
5295 : later decides the SLP representation is not good for other
5296 : store vectorization methods. */
5297 8360 : bool want_store_lanes
5298 8360 : = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5299 8360 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5300 6260 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5301 6256 : && compare_step_with_zero (vinfo, stmt_info) > 0
5302 14511 : && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
5303 16720 : masked_p, group_size, i));
5304 8360 : if (want_store_lanes || force_single_lane)
5305 : i = 1;
5306 :
5307 : /* A fatal discovery fail doesn't always mean single-lane SLP
5308 : isn't a possibility, so try. */
5309 6589 : if (i == 0)
5310 : i = 1;
5311 :
5312 8360 : if (dump_enabled_p ())
5313 891 : dump_printf_loc (MSG_NOTE, vect_location,
5314 : "Splitting SLP group at stmt %u\n", i);
5315 :
5316 : /* Analyze the stored values and pinch them together with
5317 : a permute node so we can preserve the whole store group. */
5318 8360 : auto_vec<slp_tree> rhs_nodes;
5319 8360 : poly_uint64 max_nunits = 1;
5320 :
5321 8360 : unsigned int rhs_common_nlanes = 0;
5322 8360 : unsigned int start = 0, end = i;
5323 37866 : while (start < group_size)
5324 : {
5325 29624 : gcc_assert (end - start >= 1);
5326 29624 : vec<stmt_vec_info> substmts;
5327 29624 : substmts.create (end - start);
5328 91760 : for (unsigned j = start; j < end; ++j)
5329 62136 : substmts.quick_push (scalar_stmts[j]);
5330 29624 : max_nunits = 1;
5331 29624 : node = vect_build_slp_tree (vinfo, substmts,
5332 : &max_nunits,
5333 : matches, limit, &tree_size, bst_map);
5334 29624 : if (node)
5335 : {
5336 23817 : rhs_nodes.safe_push (node);
5337 23817 : vect_update_max_nunits (&max_nunits, node->max_nunits);
5338 23817 : if (start == 0)
5339 8242 : rhs_common_nlanes = SLP_TREE_LANES (node);
5340 15575 : else if (rhs_common_nlanes != SLP_TREE_LANES (node))
5341 1399 : rhs_common_nlanes = 0;
5342 23817 : start = end;
5343 23817 : if (want_store_lanes || force_single_lane)
5344 5315 : end = start + 1;
5345 : else
5346 : end = group_size;
5347 : }
5348 : else
5349 : {
5350 5807 : substmts.release ();
5351 5807 : if (end - start == 1)
5352 : {
5353 : /* Single-lane discovery failed. Free resources. */
5354 118 : for (auto node : rhs_nodes)
5355 0 : vect_free_slp_tree (node);
5356 118 : scalar_stmts.release ();
5357 118 : if (dump_enabled_p ())
5358 17 : dump_printf_loc (MSG_NOTE, vect_location,
5359 : "SLP discovery failed\n");
5360 118 : return false;
5361 : }
5362 :
5363 : /* ??? It really happens that we soft-fail SLP
5364 : build at a mismatch but the matching part hard-fails
5365 : later. As we know we arrived here with a group
5366 : larger than one try a group of size one! */
5367 5689 : if (!matches[0])
5368 12 : end = start + 1;
5369 : else
5370 12459 : for (unsigned j = start; j < end; j++)
5371 12459 : if (!matches[j - start])
5372 : {
5373 : end = j;
5374 : break;
5375 : }
5376 : }
5377 : }
5378 :
5379 : /* Now re-assess whether we want store lanes in case the
5380 : discovery ended up producing all single-lane RHSs. */
5381 8242 : if (! want_store_lanes
5382 8242 : && rhs_common_nlanes == 1
5383 7176 : && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5384 7176 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5385 5441 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5386 5438 : && compare_step_with_zero (vinfo, stmt_info) > 0
5387 13593 : && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
5388 : group_size, masked_p)
5389 : != IFN_LAST))
5390 : want_store_lanes = true;
5391 :
5392 : /* Now we assume we can build the root SLP node from all stores. */
5393 8242 : if (want_store_lanes)
5394 : {
5395 : /* For store-lanes feed the store node with all RHS nodes
5396 : in order. */
5397 0 : node = vect_create_new_slp_node (scalar_stmts,
5398 0 : SLP_TREE_CHILDREN
5399 : (rhs_nodes[0]).length ());
5400 0 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
5401 0 : node->max_nunits = max_nunits;
5402 0 : node->ldst_lanes = true;
5403 0 : SLP_TREE_CHILDREN (node)
5404 0 : .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
5405 0 : + rhs_nodes.length () - 1);
5406 : /* First store value and possibly mask. */
5407 0 : SLP_TREE_CHILDREN (node)
5408 0 : .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
5409 : /* Rest of the store values. All mask nodes are the same,
5410 : this should be guaranteed by dataref group discovery. */
5411 0 : for (unsigned j = 1; j < rhs_nodes.length (); ++j)
5412 0 : SLP_TREE_CHILDREN (node)
5413 0 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
5414 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
5415 0 : child->refcnt++;
5416 : }
5417 : else
5418 8242 : node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts,
5419 : max_nunits);
5420 :
5421 32059 : while (!rhs_nodes.is_empty ())
5422 23817 : vect_free_slp_tree (rhs_nodes.pop ());
5423 :
5424 : /* Create a new SLP instance. */
5425 8242 : slp_instance new_instance = XNEW (class _slp_instance);
5426 8242 : SLP_INSTANCE_TREE (new_instance) = node;
5427 8242 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5428 8242 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5429 8242 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5430 8242 : SLP_INSTANCE_KIND (new_instance) = kind;
5431 8242 : new_instance->reduc_phis = NULL;
5432 8242 : new_instance->cost_vec = vNULL;
5433 8242 : new_instance->subgraph_entries = vNULL;
5434 :
5435 8242 : if (dump_enabled_p ())
5436 874 : dump_printf_loc (MSG_NOTE, vect_location,
5437 : "SLP size %u vs. limit %u.\n",
5438 : tree_size, max_tree_size);
5439 :
5440 8242 : vinfo->slp_instances.safe_push (new_instance);
5441 :
5442 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5443 : the number of SLP lanes of the root in a few places.
5444 : Verify that assumption holds. */
5445 8242 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5446 : == group_size);
5447 :
5448 8242 : if (dump_enabled_p ())
5449 : {
5450 874 : dump_printf_loc (MSG_NOTE, vect_location,
5451 : "Final SLP tree for instance %p:\n",
5452 : (void *) new_instance);
5453 874 : vect_print_slp_graph (MSG_NOTE, vect_location,
5454 : SLP_INSTANCE_TREE (new_instance));
5455 : }
5456 8242 : return true;
5457 8360 : }
5458 : else
5459 : /* Free the allocated memory. */
5460 256866 : scalar_stmts.release ();
5461 :
5462 : /* Even though the first vector did not all match, we might be able to SLP
5463 : (some) of the remainder. FORNOW ignore this possibility. */
5464 : }
5465 : else
5466 : /* Free the allocated memory. */
5467 1 : scalar_stmts.release ();
5468 :
5469 : /* Failed to SLP. */
5470 256867 : if (dump_enabled_p ())
5471 42 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
5472 : return false;
5473 : }
5474 :
5475 : /* qsort comparator ordering SLP load nodes. */
5476 :
5477 : static int
5478 2688446 : vllp_cmp (const void *a_, const void *b_)
5479 : {
5480 2688446 : const slp_tree a = *(const slp_tree *)a_;
5481 2688446 : const slp_tree b = *(const slp_tree *)b_;
5482 2688446 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
5483 2688446 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
5484 2688446 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5485 1546893 : && STMT_VINFO_GROUPED_ACCESS (b0)
5486 4173422 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5487 : {
5488 : /* Same group, order after lanes used. */
5489 348476 : if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
5490 : return 1;
5491 339497 : else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
5492 : return -1;
5493 : else
5494 : {
5495 : /* Try to order loads using the same lanes together, breaking
5496 : the tie with the lane number that first differs. */
5497 329754 : if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5498 329754 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5499 : return 0;
5500 329754 : else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
5501 329754 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5502 : return 1;
5503 325691 : else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5504 325691 : && SLP_TREE_LOAD_PERMUTATION (b).exists ())
5505 : return -1;
5506 : else
5507 : {
5508 318205 : for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
5509 318205 : if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5510 318205 : != SLP_TREE_LOAD_PERMUTATION (b)[i])
5511 : {
5512 : /* In-order lane first, that's what the above case for
5513 : no permutation does. */
5514 316893 : if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
5515 : return -1;
5516 194237 : else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
5517 : return 1;
5518 102102 : else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5519 102102 : < SLP_TREE_LOAD_PERMUTATION (b)[i])
5520 : return -1;
5521 : else
5522 : return 1;
5523 : }
5524 : return 0;
5525 : }
5526 : }
5527 : }
5528 : else /* Different groups or non-groups. */
5529 : {
5530 : /* Order groups as their first element to keep them together. */
5531 2339970 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5532 2339970 : a0 = DR_GROUP_FIRST_ELEMENT (a0);
5533 2339970 : if (STMT_VINFO_GROUPED_ACCESS (b0))
5534 2339970 : b0 = DR_GROUP_FIRST_ELEMENT (b0);
5535 2339970 : if (a0 == b0)
5536 : return 0;
5537 : /* Tie using UID. */
5538 2339850 : else if (gimple_uid (STMT_VINFO_STMT (a0))
5539 2339850 : < gimple_uid (STMT_VINFO_STMT (b0)))
5540 : return -1;
5541 : else
5542 : {
5543 1040223 : gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
5544 : != gimple_uid (STMT_VINFO_STMT (b0)));
5545 : return 1;
5546 : }
5547 : }
5548 : }
5549 :
5550 : /* Return whether if the load permutation of NODE is consecutive starting
5551 : with value START_VAL in the first element. If START_VAL is not given
5552 : the first element's value is used. */
5553 :
5554 : bool
5555 644582 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
5556 : {
5557 644582 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
5558 :
5559 644582 : if (!perm.exists () || !perm.length ())
5560 : return false;
5561 :
5562 644582 : if (start_val == UINT_MAX)
5563 79890 : start_val = perm[0];
5564 :
5565 1273012 : for (unsigned int i = 0; i < perm.length (); i++)
5566 651917 : if (perm[i] != start_val + (unsigned int) i)
5567 : return false;
5568 :
5569 : return true;
5570 : }
5571 :
5572 : /* Process the set of LOADS that are all from the same dataref group. */
5573 :
5574 : static void
5575 161864 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5576 : scalar_stmts_to_slp_tree_map_t *bst_map,
5577 : const array_slice<slp_tree> &loads,
5578 : bool force_single_lane)
5579 : {
5580 : /* We at this point want to lower without a fixed VF or vector
5581 : size in mind which means we cannot actually compute whether we
5582 : need three or more vectors for a load permutation yet. So always
5583 : lower. */
5584 161864 : stmt_vec_info first
5585 161864 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
5586 161864 : unsigned group_lanes = DR_GROUP_SIZE (first);
5587 :
5588 : /* Verify if all load permutations can be implemented with a suitably
5589 : large element load-lanes operation. */
5590 161864 : unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
5591 161864 : if (STMT_VINFO_STRIDED_P (first)
5592 159411 : || compare_step_with_zero (loop_vinfo, first) <= 0
5593 156723 : || exact_log2 (ld_lanes_lanes) == -1
5594 : /* ??? For now only support the single-lane case as there is
5595 : missing support on the store-lane side and code generation
5596 : isn't up to the task yet. */
5597 153920 : || ld_lanes_lanes != 1
5598 304757 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
5599 : group_lanes / ld_lanes_lanes,
5600 : false) == IFN_LAST)
5601 : ld_lanes_lanes = 0;
5602 : else
5603 : /* Verify the loads access the same number of lanes aligned to
5604 : ld_lanes_lanes. */
5605 0 : for (slp_tree load : loads)
5606 : {
5607 0 : if (SLP_TREE_LANES (load) != ld_lanes_lanes)
5608 : {
5609 : ld_lanes_lanes = 0;
5610 : break;
5611 : }
5612 0 : unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
5613 0 : if (first % ld_lanes_lanes != 0)
5614 : {
5615 : ld_lanes_lanes = 0;
5616 : break;
5617 : }
5618 0 : if (!vect_load_perm_consecutive_p (load))
5619 : {
5620 : ld_lanes_lanes = 0;
5621 : break;
5622 : }
5623 : }
5624 :
5625 : /* Only a power-of-two number of lanes matches interleaving with N levels.
5626 : ??? An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
5627 : at each step. */
5628 262529 : if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
5629 : return;
5630 :
5631 266550 : for (slp_tree load : loads)
5632 : {
5633 : /* Leave masked or gather loads alone for now. */
5634 188133 : if (!SLP_TREE_CHILDREN (load).is_empty ())
5635 60950 : continue;
5636 :
5637 : /* For single-element interleaving spanning multiple vectors avoid
5638 : lowering, we want to use VMAT_ELEMENTWISE later. */
5639 188127 : if (ld_lanes_lanes == 0
5640 188127 : && SLP_TREE_LANES (load) == 1
5641 168652 : && !DR_GROUP_NEXT_ELEMENT (first)
5642 267826 : && maybe_gt (group_lanes,
5643 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
5644 51309 : return;
5645 :
5646 : /* We want to pattern-match special cases here and keep those
5647 : alone. Candidates are splats and load-lane. */
5648 :
5649 : /* We need to lower only loads of less than half of the groups
5650 : lanes, including duplicate lanes. Note this leaves nodes
5651 : with a non-1:1 load permutation around instead of canonicalizing
5652 : those into a load and a permute node. Removing this early
5653 : check would do such canonicalization. */
5654 136818 : if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
5655 57361 : && ld_lanes_lanes == 0)
5656 57361 : continue;
5657 :
5658 : /* Build the permute to get the original load permutation order. */
5659 79457 : bool contiguous = vect_load_perm_consecutive_p (load);
5660 79457 : lane_permutation_t final_perm;
5661 79457 : final_perm.create (SLP_TREE_LANES (load));
5662 159858 : for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
5663 160802 : final_perm.quick_push (
5664 80401 : std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
5665 :
5666 : /* When the load permutation accesses a contiguous unpermuted,
5667 : power-of-two aligned and sized chunk leave the load alone.
5668 : We can likely (re-)load it more efficiently rather than
5669 : extracting it from the larger load.
5670 : ??? Long-term some of the lowering should move to where
5671 : the vector types involved are fixed. */
5672 83040 : if (!force_single_lane
5673 79457 : && ld_lanes_lanes == 0
5674 53582 : && contiguous
5675 53322 : && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
5676 6579 : && pow2p_hwi (SLP_TREE_LANES (load))
5677 6543 : && pow2p_hwi (group_lanes)
5678 3583 : && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
5679 83040 : && group_lanes % SLP_TREE_LANES (load) == 0)
5680 : {
5681 3583 : final_perm.release ();
5682 3583 : continue;
5683 : }
5684 :
5685 : /* First build (and possibly re-use) a load node for the
5686 : unpermuted group. Gaps in the middle and on the end are
5687 : represented with NULL stmts. */
5688 75874 : vec<stmt_vec_info> stmts;
5689 75874 : stmts.create (group_lanes);
5690 270839 : for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
5691 : {
5692 194965 : if (s != first)
5693 124202 : for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
5694 5111 : stmts.quick_push (NULL);
5695 194965 : stmts.quick_push (s);
5696 : }
5697 139813 : for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
5698 63939 : stmts.quick_push (NULL);
5699 75874 : poly_uint64 max_nunits = 1;
5700 75874 : bool *matches = XALLOCAVEC (bool, group_lanes);
5701 75874 : unsigned limit = 1;
5702 75874 : unsigned tree_size = 0;
5703 75874 : slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts,
5704 : &max_nunits, matches, &limit,
5705 75874 : &tree_size, bst_map);
5706 75874 : gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
5707 :
5708 75874 : if (ld_lanes_lanes != 0)
5709 : {
5710 : /* ??? If this is not in sync with what get_load_store_type
5711 : later decides the SLP representation is not good for other
5712 : store vectorization methods. */
5713 0 : l0->ldst_lanes = true;
5714 0 : load->ldst_lanes = true;
5715 : }
5716 :
5717 236320 : while (1)
5718 : {
5719 156097 : unsigned group_lanes = SLP_TREE_LANES (l0);
5720 156097 : if (ld_lanes_lanes != 0
5721 156097 : || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
5722 : break;
5723 :
5724 : /* Try to lower by reducing the group to half its size using an
5725 : interleaving scheme. For this try to compute whether all
5726 : elements needed for this load are in even or odd elements of
5727 : an even/odd decomposition with N consecutive elements.
5728 : Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
5729 : with N == 2. */
5730 : /* ??? Only an even number of lanes can be handed this way, but the
5731 : fallback below could work for any number. We have to make sure
5732 : to round up in that case. */
5733 80223 : gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
5734 12150 : unsigned even = 0, odd = 0;
5735 12150 : if ((group_lanes & 1) == 0)
5736 : {
5737 12150 : even = (1 << ceil_log2 (group_lanes)) - 1;
5738 12150 : odd = even;
5739 49349 : for (auto l : final_perm)
5740 : {
5741 12899 : even &= ~l.second;
5742 12899 : odd &= l.second;
5743 : }
5744 : }
5745 :
5746 : /* Now build an even or odd extraction from the unpermuted load. */
5747 80223 : lane_permutation_t perm;
5748 80223 : perm.create ((group_lanes + 1) / 2);
5749 80223 : unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
5750 80223 : unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
5751 80223 : if (even_level
5752 11168 : && group_lanes % (2 * even_level) == 0
5753 : /* ??? When code generating permutes we do not try to pun
5754 : to larger component modes so level != 1 isn't a natural
5755 : even/odd extract. Prefer one if possible. */
5756 11168 : && (even_level == 1 || !odd_level || odd_level != 1))
5757 : {
5758 : /* { 0, 1, ... 4, 5 ..., } */
5759 39574 : for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
5760 62465 : for (unsigned j = 0; j < even_level; ++j)
5761 31428 : perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
5762 : }
5763 69055 : else if (odd_level)
5764 : {
5765 : /* { ..., 2, 3, ... 6, 7 } */
5766 3583 : gcc_assert (group_lanes % (2 * odd_level) == 0);
5767 15451 : for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
5768 23790 : for (unsigned j = 0; j < odd_level; ++j)
5769 11922 : perm.quick_push
5770 11922 : (std::make_pair (0, (2 * i + 1) * odd_level + j));
5771 : }
5772 : else
5773 : {
5774 : /* As fallback extract all used lanes and fill to half the
5775 : group size by repeating the last element.
5776 : ??? This is quite a bad strathegy for re-use - we could
5777 : brute force our way to find more optimal filling lanes to
5778 : maximize re-use when looking at all loads from the group. */
5779 68103 : auto_bitmap l;
5780 272468 : for (auto p : final_perm)
5781 68159 : bitmap_set_bit (l, p.second);
5782 68103 : unsigned i = 0;
5783 68103 : bitmap_iterator bi;
5784 136262 : EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
5785 68159 : perm.quick_push (std::make_pair (0, i));
5786 272564 : while (perm.length () < (group_lanes + 1) / 2)
5787 68179 : perm.quick_push (perm.last ());
5788 68103 : }
5789 :
5790 : /* Update final_perm with the intermediate permute. */
5791 161195 : for (unsigned i = 0; i < final_perm.length (); ++i)
5792 : {
5793 80972 : unsigned l = final_perm[i].second;
5794 80972 : unsigned j;
5795 90537 : for (j = 0; j < perm.length (); ++j)
5796 90537 : if (perm[j].second == l)
5797 : {
5798 80972 : final_perm[i].second = j;
5799 80972 : break;
5800 : }
5801 80972 : gcc_assert (j < perm.length ());
5802 : }
5803 :
5804 : /* And create scalar stmts. */
5805 80223 : vec<stmt_vec_info> perm_stmts;
5806 80223 : perm_stmts.create (perm.length ());
5807 259911 : for (unsigned i = 0; i < perm.length (); ++i)
5808 179688 : perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
5809 :
5810 80223 : slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
5811 80223 : SLP_TREE_CHILDREN (p).quick_push (l0);
5812 80223 : SLP_TREE_LANE_PERMUTATION (p) = perm;
5813 80223 : SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
5814 80223 : SLP_TREE_LANES (p) = perm.length ();
5815 80223 : SLP_TREE_REPRESENTATIVE (p) = SLP_TREE_REPRESENTATIVE (load);
5816 : /* ??? As we have scalar stmts for this intermediate permute we
5817 : could CSE it via bst_map but we do not want to pick up
5818 : another SLP node with a load permutation. We instead should
5819 : have a "local" CSE map here. */
5820 80223 : SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
5821 :
5822 : /* We now have a node for (group_lanes + 1) / 2 lanes. */
5823 80223 : l0 = p;
5824 80223 : }
5825 :
5826 : /* And finally from the ordered reduction node create the
5827 : permute to shuffle the lanes into the original load-permutation
5828 : order. We replace the original load node with this. */
5829 75874 : SLP_TREE_CODE (load) = VEC_PERM_EXPR;
5830 75874 : SLP_TREE_LOAD_PERMUTATION (load).release ();
5831 75874 : SLP_TREE_LANE_PERMUTATION (load) = final_perm;
5832 75874 : SLP_TREE_CHILDREN (load).create (1);
5833 75874 : SLP_TREE_CHILDREN (load).quick_push (l0);
5834 : }
5835 : }
5836 :
5837 : /* Transform SLP loads in the SLP graph created by SLP discovery to
5838 : group loads from the same group and lower load permutations that
5839 : are unlikely to be supported into a series of permutes.
5840 : In the degenerate case of having only single-lane SLP instances
5841 : this should result in a series of permute nodes emulating an
5842 : interleaving scheme. */
5843 :
5844 : static void
5845 497099 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5846 : scalar_stmts_to_slp_tree_map_t *bst_map,
5847 : bool force_single_lane)
5848 : {
5849 : /* Gather and sort loads across all instances. */
5850 497099 : hash_set<slp_tree> visited;
5851 497099 : auto_vec<slp_tree> loads;
5852 2281435 : for (auto inst : loop_vinfo->slp_instances)
5853 792086 : vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
5854 497099 : if (loads.is_empty ())
5855 93364 : return;
5856 403735 : loads.qsort (vllp_cmp);
5857 :
5858 : /* Now process each dataref group separately. */
5859 403735 : unsigned firsti = 0;
5860 747521 : for (unsigned i = 1; i < loads.length (); ++i)
5861 : {
5862 343786 : slp_tree first = loads[firsti];
5863 343786 : slp_tree next = loads[i];
5864 343786 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
5865 343786 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
5866 343786 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5867 158736 : && STMT_VINFO_GROUPED_ACCESS (b0)
5868 489412 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5869 63482 : continue;
5870 : /* Now we have one or multiple SLP loads of the same group from
5871 : firsti to i - 1. */
5872 280304 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5873 95254 : vect_lower_load_permutations (loop_vinfo, bst_map,
5874 95254 : make_array_slice (&loads[firsti],
5875 : i - firsti),
5876 : force_single_lane);
5877 : firsti = i;
5878 : }
5879 807470 : if (firsti < loads.length ()
5880 807470 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
5881 66610 : vect_lower_load_permutations (loop_vinfo, bst_map,
5882 66610 : make_array_slice (&loads[firsti],
5883 66610 : loads.length () - firsti),
5884 : force_single_lane);
5885 497099 : }
5886 :
5887 : /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
5888 : trees of packed scalar stmts if SLP is possible. */
5889 :
5890 : opt_result
5891 1178419 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
5892 : bool force_single_lane)
5893 : {
5894 1178419 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5895 1178419 : unsigned int i;
5896 1178419 : stmt_vec_info first_element;
5897 1178419 : slp_instance instance;
5898 :
5899 1178419 : DUMP_VECT_SCOPE ("vect_analyze_slp");
5900 :
5901 1178419 : unsigned limit = max_tree_size;
5902 :
5903 1178419 : scalar_stmts_to_slp_tree_map_t *bst_map
5904 1178419 : = new scalar_stmts_to_slp_tree_map_t ();
5905 :
5906 : /* Find SLP sequences starting from groups of grouped stores. */
5907 3292514 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
5908 935802 : if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
5909 : slp_inst_kind_store, max_tree_size, &limit,
5910 : force_single_lane)
5911 935802 : && loop_vinfo)
5912 : {
5913 126 : release_scalar_stmts_to_slp_tree_map (bst_map);
5914 126 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5915 : }
5916 :
5917 : /* For loops also start SLP discovery from non-grouped stores. */
5918 1178293 : if (loop_vinfo)
5919 : {
5920 : data_reference_p dr;
5921 1678416 : FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
5922 1167124 : if (DR_IS_WRITE (dr))
5923 : {
5924 378480 : stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
5925 : /* Grouped stores are already handled above. */
5926 378480 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5927 102733 : continue;
5928 275747 : vec<stmt_vec_info> stmts;
5929 275747 : vec<stmt_vec_info> roots = vNULL;
5930 275747 : vec<tree> remain = vNULL;
5931 275747 : stmts.create (1);
5932 275747 : stmts.quick_push (stmt_info);
5933 275747 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5934 : stmts, roots, remain, max_tree_size,
5935 : &limit, bst_map, force_single_lane))
5936 : {
5937 3559 : release_scalar_stmts_to_slp_tree_map (bst_map);
5938 3559 : return opt_result::failure_at (vect_location,
5939 : "SLP build failed.\n");
5940 : }
5941 : }
5942 :
5943 : stmt_vec_info stmt_info;
5944 511332 : FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
5945 : {
5946 20 : vec<stmt_vec_info> stmts;
5947 20 : vec<stmt_vec_info> roots = vNULL;
5948 20 : vec<tree> remain = vNULL;
5949 20 : stmts.create (1);
5950 20 : stmts.quick_push (stmt_info);
5951 20 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5952 : stmts, roots, remain, max_tree_size,
5953 : &limit, bst_map, force_single_lane))
5954 : {
5955 0 : release_scalar_stmts_to_slp_tree_map (bst_map);
5956 0 : return opt_result::failure_at (vect_location,
5957 : "SLP build failed.\n");
5958 : }
5959 : }
5960 : }
5961 :
5962 1174734 : if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
5963 : {
5964 2043380 : for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
5965 : {
5966 1379938 : vect_location = bb_vinfo->roots[i].roots[0]->stmt;
5967 : /* Apply patterns. */
5968 4521544 : for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
5969 6283212 : bb_vinfo->roots[i].stmts[j]
5970 3235846 : = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
5971 1379938 : if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
5972 1379938 : bb_vinfo->roots[i].stmts,
5973 1379938 : bb_vinfo->roots[i].roots,
5974 1379938 : bb_vinfo->roots[i].remain,
5975 : max_tree_size, &limit, bst_map, false))
5976 : {
5977 154635 : bb_vinfo->roots[i].roots = vNULL;
5978 154635 : bb_vinfo->roots[i].remain = vNULL;
5979 : }
5980 1379938 : bb_vinfo->roots[i].stmts = vNULL;
5981 : }
5982 : }
5983 :
5984 1174734 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
5985 : {
5986 : /* Find SLP sequences starting from groups of reductions. */
5987 511292 : if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
5988 : bst_map, force_single_lane))
5989 : {
5990 1900 : release_scalar_stmts_to_slp_tree_map (bst_map);
5991 1900 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5992 : }
5993 :
5994 : /* Make sure to vectorize only-live stmts, usually inductions. */
5995 2294805 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
5996 1482870 : for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
5997 706562 : gsi_next (&gsi))
5998 : {
5999 716241 : gphi *lc_phi = *gsi;
6000 716241 : tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
6001 716241 : stmt_vec_info stmt_info;
6002 716241 : if (TREE_CODE (def) == SSA_NAME
6003 601364 : && !virtual_operand_p (def)
6004 308250 : && (stmt_info = loop_vinfo->lookup_def (def))
6005 276513 : && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
6006 276513 : && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
6007 215538 : && STMT_VINFO_LIVE_P (stmt_info)
6008 215538 : && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
6009 827781 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
6010 : {
6011 111452 : vec<stmt_vec_info> stmts;
6012 111452 : vec<stmt_vec_info> roots = vNULL;
6013 111452 : vec<tree> remain = vNULL;
6014 111452 : stmts.create (1);
6015 111452 : stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
6016 111452 : if (! vect_build_slp_instance (vinfo,
6017 : slp_inst_kind_reduc_group,
6018 : stmts, roots, remain,
6019 : max_tree_size, &limit,
6020 : bst_map, force_single_lane))
6021 : {
6022 9679 : release_scalar_stmts_to_slp_tree_map (bst_map);
6023 9679 : return opt_result::failure_at (vect_location,
6024 : "SLP build failed.\n");
6025 : }
6026 : }
6027 9679 : }
6028 :
6029 : /* Find SLP sequences starting from gconds. */
6030 1249946 : for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
6031 : {
6032 291793 : auto cond_info = loop_vinfo->lookup_stmt (cond);
6033 :
6034 291793 : cond_info = vect_stmt_to_vectorize (cond_info);
6035 291793 : vec<stmt_vec_info> roots = vNULL;
6036 291793 : roots.safe_push (cond_info);
6037 291793 : gimple *stmt = STMT_VINFO_STMT (cond_info);
6038 291793 : tree args0 = gimple_cond_lhs (stmt);
6039 291793 : tree args1 = gimple_cond_rhs (stmt);
6040 :
6041 : /* These should be enforced by cond lowering, but if it failed
6042 : bail. */
6043 291793 : if (gimple_cond_code (stmt) != NE_EXPR
6044 290653 : || TREE_TYPE (args0) != boolean_type_node
6045 581870 : || !integer_zerop (args1))
6046 : {
6047 1716 : roots.release ();
6048 1716 : release_scalar_stmts_to_slp_tree_map (bst_map);
6049 1716 : return opt_result::failure_at (vect_location,
6050 : "SLP build failed.\n");
6051 : }
6052 :
6053 : /* An argument without a loop def will be codegened from vectorizing the
6054 : root gcond itself. As such we don't need to try to build an SLP tree
6055 : from them. It's highly likely that the resulting SLP tree here if both
6056 : arguments have a def will be incompatible, but we rely on it being split
6057 : later on. */
6058 290077 : auto varg = loop_vinfo->lookup_def (args0);
6059 290077 : vec<stmt_vec_info> stmts;
6060 290077 : vec<tree> remain = vNULL;
6061 290077 : stmts.create (1);
6062 290077 : stmts.quick_push (vect_stmt_to_vectorize (varg));
6063 :
6064 290077 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
6065 : stmts, roots, remain,
6066 : max_tree_size, &limit,
6067 : bst_map, force_single_lane))
6068 : {
6069 898 : roots.release ();
6070 898 : release_scalar_stmts_to_slp_tree_map (bst_map);
6071 898 : return opt_result::failure_at (vect_location,
6072 : "SLP build failed.\n");
6073 : }
6074 : }
6075 : }
6076 :
6077 1160541 : hash_set<slp_tree> visited_patterns;
6078 1160541 : slp_tree_to_load_perm_map_t perm_cache;
6079 1160541 : slp_compat_nodes_map_t compat_cache;
6080 :
6081 : /* See if any patterns can be found in the SLP tree. */
6082 1160541 : bool pattern_found = false;
6083 3939111 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6084 1618029 : pattern_found |= vect_match_slp_patterns (instance, vinfo,
6085 : &visited_patterns, &perm_cache,
6086 : &compat_cache);
6087 :
6088 : /* If any were found optimize permutations of loads. */
6089 1160541 : if (pattern_found)
6090 : {
6091 265 : hash_map<slp_tree, slp_tree> load_map;
6092 4705 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6093 : {
6094 4175 : slp_tree root = SLP_INSTANCE_TREE (instance);
6095 4175 : optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
6096 : &load_map, root);
6097 : }
6098 265 : }
6099 :
6100 : /* Check whether we should force some SLP instances to use load/store-lanes
6101 : and do so by forcing SLP re-discovery with single lanes. We used
6102 : to cancel SLP when this applied to all instances in a loop but now
6103 : we decide this per SLP instance. It's important to do this only
6104 : after SLP pattern recognition. */
6105 1160541 : if (is_a <loop_vec_info> (vinfo))
6106 1289185 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6107 792086 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
6108 300767 : && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
6109 : {
6110 300767 : slp_tree slp_root = SLP_INSTANCE_TREE (instance);
6111 300767 : unsigned int group_size = SLP_TREE_LANES (slp_root);
6112 300767 : tree vectype = SLP_TREE_VECTYPE (slp_root);
6113 :
6114 300767 : stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
6115 300767 : gimple *rep = STMT_VINFO_STMT (rep_info);
6116 300767 : bool masked = (is_gimple_call (rep)
6117 2446 : && gimple_call_internal_p (rep)
6118 303193 : && internal_fn_mask_index
6119 2426 : (gimple_call_internal_fn (rep)) != -1);
6120 300747 : if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
6121 30000 : || slp_root->ldst_lanes
6122 330767 : || (vect_store_lanes_supported (vectype, group_size, masked)
6123 : == IFN_LAST))
6124 300767 : continue;
6125 :
6126 0 : auto_vec<slp_tree> loads;
6127 0 : hash_set<slp_tree> visited;
6128 0 : vect_gather_slp_loads (loads, slp_root, visited);
6129 :
6130 : /* Check whether any load in the SLP instance is possibly
6131 : permuted. */
6132 0 : bool loads_permuted = false;
6133 0 : slp_tree load_node;
6134 0 : unsigned j;
6135 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6136 : {
6137 0 : if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
6138 0 : continue;
6139 0 : for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
6140 0 : if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
6141 : {
6142 : loads_permuted = true;
6143 : break;
6144 : }
6145 : }
6146 :
6147 : /* If the loads and stores can use load/store-lanes force re-discovery
6148 : with single lanes. */
6149 0 : if (loads_permuted)
6150 : {
6151 0 : bool can_use_lanes = true;
6152 : bool prefer_load_lanes = false;
6153 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6154 0 : if (STMT_VINFO_GROUPED_ACCESS
6155 : (SLP_TREE_REPRESENTATIVE (load_node)))
6156 : {
6157 0 : stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
6158 : (SLP_TREE_REPRESENTATIVE (load_node));
6159 0 : rep = STMT_VINFO_STMT (stmt_vinfo);
6160 0 : masked = (is_gimple_call (rep)
6161 0 : && gimple_call_internal_p (rep)
6162 0 : && internal_fn_mask_index
6163 0 : (gimple_call_internal_fn (rep)));
6164 : /* Use SLP for strided accesses (or if we can't
6165 : load-lanes). */
6166 0 : if (STMT_VINFO_STRIDED_P (stmt_vinfo)
6167 0 : || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
6168 0 : || vect_load_lanes_supported
6169 0 : (SLP_TREE_VECTYPE (load_node),
6170 0 : DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
6171 : /* ??? During SLP re-discovery with a single lane
6172 : a masked grouped load will appear permuted and
6173 : discovery will fail. We have to rework this
6174 : on the discovery side - for now avoid ICEing. */
6175 0 : || masked)
6176 : {
6177 : can_use_lanes = false;
6178 : break;
6179 : }
6180 : /* Make sure that the target would prefer store-lanes
6181 : for at least one of the loads.
6182 :
6183 : ??? Perhaps we should instead require this for
6184 : all loads? */
6185 0 : prefer_load_lanes
6186 : = (prefer_load_lanes
6187 0 : || SLP_TREE_LANES (load_node) == group_size
6188 0 : || (vect_slp_prefer_store_lanes_p
6189 0 : (vinfo, stmt_vinfo,
6190 : SLP_TREE_VECTYPE (load_node), masked,
6191 : group_size, SLP_TREE_LANES (load_node))));
6192 : }
6193 :
6194 0 : if (can_use_lanes && prefer_load_lanes)
6195 : {
6196 0 : if (dump_enabled_p ())
6197 0 : dump_printf_loc (MSG_NOTE, vect_location,
6198 : "SLP instance %p can use load/store-lanes,"
6199 : " re-discovering with single-lanes\n",
6200 : (void *) instance);
6201 :
6202 0 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
6203 :
6204 0 : vect_free_slp_instance (instance);
6205 0 : limit = max_tree_size;
6206 0 : bool res = vect_analyze_slp_instance (vinfo, bst_map,
6207 : stmt_info,
6208 : slp_inst_kind_store,
6209 : max_tree_size, &limit,
6210 : true);
6211 0 : gcc_assert (res);
6212 0 : auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
6213 0 : LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
6214 : }
6215 : }
6216 0 : }
6217 :
6218 : /* When we end up with load permutations that we cannot possibly handle,
6219 : like those requiring three vector inputs, lower them using interleaving
6220 : like schemes. */
6221 1160541 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6222 : {
6223 497099 : vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
6224 497099 : if (dump_enabled_p ())
6225 : {
6226 20590 : dump_printf_loc (MSG_NOTE, vect_location,
6227 : "SLP graph after lowering permutations:\n");
6228 20590 : hash_set<slp_tree> visited;
6229 91611 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6230 29866 : vect_print_slp_graph (MSG_NOTE, vect_location,
6231 : SLP_INSTANCE_TREE (instance), visited);
6232 20590 : }
6233 : }
6234 :
6235 1160541 : release_scalar_stmts_to_slp_tree_map (bst_map);
6236 :
6237 1160541 : if (pattern_found && dump_enabled_p ())
6238 : {
6239 18 : dump_printf_loc (MSG_NOTE, vect_location,
6240 : "Pattern matched SLP tree\n");
6241 18 : hash_set<slp_tree> visited;
6242 91 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6243 37 : vect_print_slp_graph (MSG_NOTE, vect_location,
6244 : SLP_INSTANCE_TREE (instance), visited);
6245 18 : }
6246 :
6247 1160541 : return opt_result::success ();
6248 1160541 : }
6249 :
6250 : /* Estimates the cost of inserting layout changes into the SLP graph.
6251 : It can also say that the insertion is impossible. */
6252 :
6253 : struct slpg_layout_cost
6254 : {
6255 11181218 : slpg_layout_cost () = default;
6256 : slpg_layout_cost (sreal, bool);
6257 :
6258 535279 : static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
6259 5706051 : bool is_possible () const { return depth != sreal::max (); }
6260 :
6261 : bool operator== (const slpg_layout_cost &) const;
6262 : bool operator!= (const slpg_layout_cost &) const;
6263 :
6264 : bool is_better_than (const slpg_layout_cost &, bool) const;
6265 :
6266 : void add_parallel_cost (const slpg_layout_cost &);
6267 : void add_serial_cost (const slpg_layout_cost &);
6268 : void split (unsigned int);
6269 :
6270 : /* The longest sequence of layout changes needed during any traversal
6271 : of the partition dag, weighted by execution frequency.
6272 :
6273 : This is the most important metric when optimizing for speed, since
6274 : it helps to ensure that we keep the number of operations on
6275 : critical paths to a minimum. */
6276 : sreal depth = 0;
6277 :
6278 : /* An estimate of the total number of operations needed. It is weighted by
6279 : execution frequency when optimizing for speed but not when optimizing for
6280 : size. In order to avoid double-counting, a node with a fanout of N will
6281 : distribute 1/N of its total cost to each successor.
6282 :
6283 : This is the most important metric when optimizing for size, since
6284 : it helps to keep the total number of operations to a minimum, */
6285 : sreal total = 0;
6286 : };
6287 :
6288 : /* Construct costs for a node with weight WEIGHT. A higher weight
6289 : indicates more frequent execution. IS_FOR_SIZE is true if we are
6290 : optimizing for size rather than speed. */
6291 :
6292 1345231 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
6293 1346350 : : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
6294 : {
6295 1345231 : }
6296 :
6297 : bool
6298 0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
6299 : {
6300 0 : return depth == other.depth && total == other.total;
6301 : }
6302 :
6303 : bool
6304 0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
6305 : {
6306 0 : return !operator== (other);
6307 : }
6308 :
6309 : /* Return true if these costs are better than OTHER. IS_FOR_SIZE is
6310 : true if we are optimizing for size rather than speed. */
6311 :
6312 : bool
6313 326727 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
6314 : bool is_for_size) const
6315 : {
6316 326727 : if (is_for_size)
6317 : {
6318 528 : if (total != other.total)
6319 236 : return total < other.total;
6320 292 : return depth < other.depth;
6321 : }
6322 : else
6323 : {
6324 326199 : if (depth != other.depth)
6325 134573 : return depth < other.depth;
6326 191626 : return total < other.total;
6327 : }
6328 : }
6329 :
6330 : /* Increase the costs to account for something with cost INPUT_COST
6331 : happening in parallel with the current costs. */
6332 :
6333 : void
6334 389894 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
6335 : {
6336 389894 : depth = std::max (depth, input_cost.depth);
6337 389894 : total += input_cost.total;
6338 389894 : }
6339 :
6340 : /* Increase the costs to account for something with cost INPUT_COST
6341 : happening in series with the current costs. */
6342 :
6343 : void
6344 1600819 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
6345 : {
6346 1600819 : depth += other.depth;
6347 1600819 : total += other.total;
6348 1600819 : }
6349 :
6350 : /* Split the total cost among TIMES successors or predecessors. */
6351 :
6352 : void
6353 1299260 : slpg_layout_cost::split (unsigned int times)
6354 : {
6355 1299260 : if (times > 1)
6356 607052 : total /= times;
6357 1299260 : }
6358 :
6359 : /* Information about one node in the SLP graph, for use during
6360 : vect_optimize_slp_pass. */
6361 :
6362 : struct slpg_vertex
6363 : {
6364 10449829 : slpg_vertex (slp_tree node_) : node (node_) {}
6365 :
6366 : /* The node itself. */
6367 : slp_tree node;
6368 :
6369 : /* Which partition the node belongs to, or -1 if none. Nodes outside of
6370 : partitions are flexible; they can have whichever layout consumers
6371 : want them to have. */
6372 : int partition = -1;
6373 :
6374 : /* The number of nodes that directly use the result of this one
6375 : (i.e. the number of nodes that count this one as a child). */
6376 : unsigned int out_degree = 0;
6377 :
6378 : /* The execution frequency of the node. */
6379 : sreal weight = 0;
6380 :
6381 : /* The total execution frequency of all nodes that directly use the
6382 : result of this one. */
6383 : sreal out_weight = 0;
6384 : };
6385 :
6386 : /* Information about one partition of the SLP graph, for use during
6387 : vect_optimize_slp_pass. */
6388 :
6389 : struct slpg_partition_info
6390 : {
6391 : /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
6392 : of m_partitioned_nodes. */
6393 : unsigned int node_begin = 0;
6394 : unsigned int node_end = 0;
6395 :
6396 : /* Which layout we've chosen to use for this partition, or -1 if
6397 : we haven't picked one yet. */
6398 : int layout = -1;
6399 :
6400 : /* The number of predecessors and successors in the partition dag.
6401 : The predecessors always have lower partition numbers and the
6402 : successors always have higher partition numbers.
6403 :
6404 : Note that the directions of these edges are not necessarily the
6405 : same as in the data flow graph. For example, if an SCC has separate
6406 : partitions for an inner loop and an outer loop, the inner loop's
6407 : partition will have at least two incoming edges from the outer loop's
6408 : partition: one for a live-in value and one for a live-out value.
6409 : In data flow terms, one of these edges would also be from the outer loop
6410 : to the inner loop, but the other would be in the opposite direction. */
6411 : unsigned int in_degree = 0;
6412 : unsigned int out_degree = 0;
6413 : };
6414 :
6415 : /* Information about the costs of using a particular layout for a
6416 : particular partition. It can also say that the combination is
6417 : impossible. */
6418 :
6419 : struct slpg_partition_layout_costs
6420 : {
6421 1669832 : bool is_possible () const { return internal_cost.is_possible (); }
6422 67581 : void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
6423 :
6424 : /* The costs inherited from predecessor partitions. */
6425 : slpg_layout_cost in_cost;
6426 :
6427 : /* The inherent cost of the layout within the node itself. For example,
6428 : this is nonzero for a load if choosing a particular layout would require
6429 : the load to permute the loaded elements. It is nonzero for a
6430 : VEC_PERM_EXPR if the permutation cannot be eliminated or converted
6431 : to full-vector moves. */
6432 : slpg_layout_cost internal_cost;
6433 :
6434 : /* The costs inherited from successor partitions. */
6435 : slpg_layout_cost out_cost;
6436 : };
6437 :
6438 : /* This class tries to optimize the layout of vectors in order to avoid
6439 : unnecessary shuffling. At the moment, the set of possible layouts are
6440 : restricted to bijective permutations.
6441 :
6442 : The goal of the pass depends on whether we're optimizing for size or
6443 : for speed. When optimizing for size, the goal is to reduce the overall
6444 : number of layout changes (including layout changes implied by things
6445 : like load permutations). When optimizing for speed, the goal is to
6446 : reduce the maximum latency attributable to layout changes on any
6447 : non-cyclical path through the data flow graph.
6448 :
6449 : For example, when optimizing a loop nest for speed, we will prefer
6450 : to make layout changes outside of a loop rather than inside of a loop,
6451 : and will prefer to make layout changes in parallel rather than serially,
6452 : even if that increases the overall number of layout changes.
6453 :
6454 : The high-level procedure is:
6455 :
6456 : (1) Build a graph in which edges go from uses (parents) to definitions
6457 : (children).
6458 :
6459 : (2) Divide the graph into a dag of strongly-connected components (SCCs).
6460 :
6461 : (3) When optimizing for speed, partition the nodes in each SCC based
6462 : on their containing cfg loop. When optimizing for size, treat
6463 : each SCC as a single partition.
6464 :
6465 : This gives us a dag of partitions. The goal is now to assign a
6466 : layout to each partition.
6467 :
6468 : (4) Construct a set of vector layouts that are worth considering.
6469 : Record which nodes must keep their current layout.
6470 :
6471 : (5) Perform a forward walk over the partition dag (from loads to stores)
6472 : accumulating the "forward" cost of using each layout. When visiting
6473 : each partition, assign a tentative choice of layout to the partition
6474 : and use that choice when calculating the cost of using a different
6475 : layout in successor partitions.
6476 :
6477 : (6) Perform a backward walk over the partition dag (from stores to loads),
6478 : accumulating the "backward" cost of using each layout. When visiting
6479 : each partition, make a final choice of layout for that partition based
6480 : on the accumulated forward costs (from (5)) and backward costs
6481 : (from (6)).
6482 :
6483 : (7) Apply the chosen layouts to the SLP graph.
6484 :
6485 : For example, consider the SLP statements:
6486 :
6487 : S1: a_1 = load
6488 : loop:
6489 : S2: a_2 = PHI<a_1, a_3>
6490 : S3: b_1 = load
6491 : S4: a_3 = a_2 + b_1
6492 : exit:
6493 : S5: a_4 = PHI<a_3>
6494 : S6: store a_4
6495 :
6496 : S2 and S4 form an SCC and are part of the same loop. Every other
6497 : statement is in a singleton SCC. In this example there is a one-to-one
6498 : mapping between SCCs and partitions and the partition dag looks like this;
6499 :
6500 : S1 S3
6501 : \ /
6502 : S2+S4
6503 : |
6504 : S5
6505 : |
6506 : S6
6507 :
6508 : S2, S3 and S4 will have a higher execution frequency than the other
6509 : statements, so when optimizing for speed, the goal is to avoid any
6510 : layout changes:
6511 :
6512 : - within S3
6513 : - within S2+S4
6514 : - on the S3->S2+S4 edge
6515 :
6516 : For example, if S3 was originally a reversing load, the goal of the
6517 : pass is to make it an unreversed load and change the layout on the
6518 : S1->S2+S4 and S2+S4->S5 edges to compensate. (Changing the layout
6519 : on S1->S2+S4 and S5->S6 would also be acceptable.)
6520 :
6521 : The difference between SCCs and partitions becomes important if we
6522 : add an outer loop:
6523 :
6524 : S1: a_1 = ...
6525 : loop1:
6526 : S2: a_2 = PHI<a_1, a_6>
6527 : S3: b_1 = load
6528 : S4: a_3 = a_2 + b_1
6529 : loop2:
6530 : S5: a_4 = PHI<a_3, a_5>
6531 : S6: c_1 = load
6532 : S7: a_5 = a_4 + c_1
6533 : exit2:
6534 : S8: a_6 = PHI<a_5>
6535 : S9: store a_6
6536 : exit1:
6537 :
6538 : Here, S2, S4, S5, S7 and S8 form a single SCC. However, when optimizing
6539 : for speed, we usually do not want restrictions in the outer loop to "infect"
6540 : the decision for the inner loop. For example, if an outer-loop node
6541 : in the SCC contains a statement with a fixed layout, that should not
6542 : prevent the inner loop from using a different layout. Conversely,
6543 : the inner loop should not dictate a layout to the outer loop: if the
6544 : outer loop does a lot of computation, then it may not be efficient to
6545 : do all of that computation in the inner loop's preferred layout.
6546 :
6547 : So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
6548 : and S5+S7 (inner). We also try to arrange partitions so that:
6549 :
6550 : - the partition for an outer loop comes before the partition for
6551 : an inner loop
6552 :
6553 : - if a sibling loop A dominates a sibling loop B, A's partition
6554 : comes before B's
6555 :
6556 : This gives the following partition dag for the example above:
6557 :
6558 : S1 S3
6559 : \ /
6560 : S2+S4+S8 S6
6561 : | \\ /
6562 : | S5+S7
6563 : |
6564 : S9
6565 :
6566 : There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
6567 : one for a reversal of the edge S7->S8.
6568 :
6569 : The backward walk picks a layout for S5+S7 before S2+S4+S8. The choice
6570 : for S2+S4+S8 therefore has to balance the cost of using the outer loop's
6571 : preferred layout against the cost of changing the layout on entry to the
6572 : inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
6573 :
6574 : Although this works well when optimizing for speed, it has the downside
6575 : when optimizing for size that the choice of layout for S5+S7 is completely
6576 : independent of S9, which lessens the chance of reducing the overall number
6577 : of permutations. We therefore do not partition SCCs when optimizing
6578 : for size.
6579 :
6580 : To give a concrete example of the difference between optimizing
6581 : for size and speed, consider:
6582 :
6583 : a[0] = (b[1] << c[3]) - d[1];
6584 : a[1] = (b[0] << c[2]) - d[0];
6585 : a[2] = (b[3] << c[1]) - d[3];
6586 : a[3] = (b[2] << c[0]) - d[2];
6587 :
6588 : There are three different layouts here: one for a, one for b and d,
6589 : and one for c. When optimizing for speed it is better to permute each
6590 : of b, c and d into the order required by a, since those permutations
6591 : happen in parallel. But when optimizing for size, it is better to:
6592 :
6593 : - permute c into the same order as b
6594 : - do the arithmetic
6595 : - permute the result into the order required by a
6596 :
6597 : This gives 2 permutations rather than 3. */
6598 :
6599 : class vect_optimize_slp_pass
6600 : {
6601 : public:
6602 709983 : vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
6603 : void run ();
6604 :
6605 : private:
6606 : /* Graph building. */
6607 : struct loop *containing_loop (slp_tree);
6608 : bool is_cfg_latch_edge (graph_edge *);
6609 : void build_vertices (hash_set<slp_tree> &, slp_tree);
6610 : void build_vertices ();
6611 : void build_graph ();
6612 :
6613 : /* Partitioning. */
6614 : void create_partitions ();
6615 : template<typename T> void for_each_partition_edge (unsigned int, T);
6616 :
6617 : /* Layout selection. */
6618 : bool is_compatible_layout (slp_tree, unsigned int);
6619 : bool is_compatible_layout (const slpg_partition_info &, unsigned int);
6620 : int change_layout_cost (slp_tree, unsigned int, unsigned int);
6621 : slpg_partition_layout_costs &partition_layout_costs (unsigned int,
6622 : unsigned int);
6623 : void change_vec_perm_layout (slp_tree, lane_permutation_t &,
6624 : int, unsigned int);
6625 : int internal_node_cost (slp_tree, int, unsigned int);
6626 : void start_choosing_layouts ();
6627 : bool legitimize ();
6628 :
6629 : /* Cost propagation. */
6630 : slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
6631 : unsigned int, unsigned int);
6632 : slpg_layout_cost total_in_cost (unsigned int);
6633 : slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
6634 : slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
6635 : void forward_pass ();
6636 : void backward_pass ();
6637 :
6638 : /* Rematerialization. */
6639 : slp_tree get_result_with_layout (slp_tree, unsigned int);
6640 : void materialize ();
6641 :
6642 : /* Clean-up. */
6643 : void remove_redundant_permutations ();
6644 :
6645 : /* Masked load lanes discovery. */
6646 : void decide_masked_load_lanes ();
6647 :
6648 : void dump ();
6649 :
6650 : vec_info *m_vinfo;
6651 :
6652 : /* True if we should optimize the graph for size, false if we should
6653 : optimize it for speed. (It wouldn't be easy to make this decision
6654 : more locally.) */
6655 : bool m_optimize_size;
6656 :
6657 : /* A graph of all SLP nodes, with edges leading from uses to definitions.
6658 : In other words, a node's predecessors are its slp_tree parents and
6659 : a node's successors are its slp_tree children. */
6660 : graph *m_slpg = nullptr;
6661 :
6662 : /* The vertices of M_SLPG, indexed by slp_tree::vertex. */
6663 : auto_vec<slpg_vertex> m_vertices;
6664 :
6665 : /* The list of all leaves of M_SLPG. such as external definitions, constants,
6666 : and loads. */
6667 : auto_vec<int> m_leafs;
6668 :
6669 : /* This array has one entry for every vector layout that we're considering.
6670 : Element 0 is null and indicates "no change". Other entries describe
6671 : permutations that are inherent in the current graph and that we would
6672 : like to reverse if possible.
6673 :
6674 : For example, a permutation { 1, 2, 3, 0 } means that something has
6675 : effectively been permuted in that way, such as a load group
6676 : { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
6677 : We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
6678 : in order to put things "back" in order. */
6679 : auto_vec<vec<unsigned> > m_perms;
6680 :
6681 : /* A partitioning of the nodes for which a layout must be chosen.
6682 : Each partition represents an <SCC, cfg loop> pair; that is,
6683 : nodes in different SCCs belong to different partitions, and nodes
6684 : within an SCC can be further partitioned according to a containing
6685 : cfg loop. Partition <SCC1, L1> comes before <SCC2, L2> if:
6686 :
6687 : - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
6688 : from leaves (such as loads) to roots (such as stores).
6689 :
6690 : - SCC1 == SCC2 and L1's header strictly dominates L2's header. */
6691 : auto_vec<slpg_partition_info> m_partitions;
6692 :
6693 : /* The list of all nodes for which a layout must be chosen. Nodes for
6694 : partition P come before the nodes for partition P+1. Nodes within a
6695 : partition are in reverse postorder. */
6696 : auto_vec<unsigned int> m_partitioned_nodes;
6697 :
6698 : /* Index P * num-layouts + L contains the cost of using layout L
6699 : for partition P. */
6700 : auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
6701 :
6702 : /* Index N * num-layouts + L, if nonnull, is a node that provides the
6703 : original output of node N adjusted to have layout L. */
6704 : auto_vec<slp_tree> m_node_layouts;
6705 : };
6706 :
6707 : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
6708 : Also record whether we should optimize anything for speed rather
6709 : than size. */
6710 :
6711 : void
6712 11290499 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
6713 : slp_tree node)
6714 : {
6715 11290499 : unsigned i;
6716 11290499 : slp_tree child;
6717 :
6718 11290499 : if (visited.add (node))
6719 11290499 : return;
6720 :
6721 10449829 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
6722 : {
6723 8253048 : basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
6724 7312102 : if (optimize_bb_for_speed_p (bb))
6725 7181954 : m_optimize_size = false;
6726 : }
6727 :
6728 10449829 : node->vertex = m_vertices.length ();
6729 10449829 : m_vertices.safe_push (slpg_vertex (node));
6730 :
6731 10449829 : bool leaf = true;
6732 10449829 : bool force_leaf = false;
6733 19599890 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
6734 9150061 : if (child)
6735 : {
6736 8239777 : leaf = false;
6737 8239777 : build_vertices (visited, child);
6738 : }
6739 : else
6740 : force_leaf = true;
6741 : /* Since SLP discovery works along use-def edges all cycles have an
6742 : entry - but there's the exception of cycles where we do not handle
6743 : the entry explicitly (but with a NULL SLP node), like some reductions
6744 : and inductions. Force those SLP PHIs to act as leafs to make them
6745 : backwards reachable. */
6746 10449829 : if (leaf || force_leaf)
6747 5175264 : m_leafs.safe_push (node->vertex);
6748 : }
6749 :
6750 : /* Fill the vertices and leafs vector with all nodes in the SLP graph. */
6751 :
6752 : void
6753 1419966 : vect_optimize_slp_pass::build_vertices ()
6754 : {
6755 1419966 : hash_set<slp_tree> visited;
6756 1419966 : unsigned i;
6757 1419966 : slp_instance instance;
6758 1419966 : m_vertices.truncate (0);
6759 1419966 : m_leafs.truncate (0);
6760 7310620 : FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
6761 3050722 : build_vertices (visited, SLP_INSTANCE_TREE (instance));
6762 1419966 : }
6763 :
6764 : /* Apply (reverse) bijectite PERM to VEC. */
6765 :
6766 : template <class T>
6767 : static void
6768 226657 : vect_slp_permute (vec<unsigned> perm,
6769 : vec<T> &vec, bool reverse)
6770 : {
6771 226657 : auto_vec<T, 64> saved;
6772 226657 : saved.create (vec.length ());
6773 755853 : for (unsigned i = 0; i < vec.length (); ++i)
6774 529196 : saved.quick_push (vec[i]);
6775 :
6776 226657 : if (reverse)
6777 : {
6778 1495825 : for (unsigned i = 0; i < vec.length (); ++i)
6779 527236 : vec[perm[i]] = saved[i];
6780 752993 : for (unsigned i = 0; i < vec.length (); ++i)
6781 907885 : gcc_assert (vec[perm[i]] == saved[i]);
6782 : }
6783 : else
6784 : {
6785 5720 : for (unsigned i = 0; i < vec.length (); ++i)
6786 1960 : vec[i] = saved[perm[i]];
6787 228617 : for (unsigned i = 0; i < vec.length (); ++i)
6788 2940 : gcc_assert (vec[i] == saved[perm[i]]);
6789 : }
6790 226657 : }
6791 :
6792 : /* Return the cfg loop that contains NODE. */
6793 :
6794 : struct loop *
6795 4069839 : vect_optimize_slp_pass::containing_loop (slp_tree node)
6796 : {
6797 4069839 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
6798 4069839 : if (!rep)
6799 6294 : return ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father;
6800 4547131 : return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
6801 : }
6802 :
6803 : /* Return true if UD (an edge from a use to a definition) is associated
6804 : with a loop latch edge in the cfg. */
6805 :
6806 : bool
6807 8239777 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
6808 : {
6809 8239777 : slp_tree use = m_vertices[ud->src].node;
6810 8239777 : slp_tree def = m_vertices[ud->dest].node;
6811 8239777 : if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
6812 8239777 : || SLP_TREE_PERMUTE_P (use))
6813 7919804 : || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
6814 : return false;
6815 :
6816 4797114 : stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
6817 4797114 : return (is_a<gphi *> (use_rep->stmt)
6818 388448 : && bb_loop_header_p (gimple_bb (use_rep->stmt))
6819 5014852 : && containing_loop (def) == containing_loop (use));
6820 : }
6821 :
6822 : /* Build the graph. Mark edges that correspond to cfg loop latch edges with
6823 : a nonnull data field. */
6824 :
6825 : void
6826 1419966 : vect_optimize_slp_pass::build_graph ()
6827 : {
6828 1419966 : m_optimize_size = true;
6829 1419966 : build_vertices ();
6830 :
6831 2839932 : m_slpg = new_graph (m_vertices.length ());
6832 14709727 : for (slpg_vertex &v : m_vertices)
6833 31243864 : for (slp_tree child : SLP_TREE_CHILDREN (v.node))
6834 9150061 : if (child)
6835 : {
6836 8239777 : graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
6837 8239777 : if (is_cfg_latch_edge (ud))
6838 207974 : ud->data = this;
6839 : }
6840 1419966 : }
6841 :
6842 : /* Return true if E corresponds to a loop latch edge in the cfg. */
6843 :
6844 : static bool
6845 4223338 : skip_cfg_latch_edges (graph_edge *e)
6846 : {
6847 4223338 : return e->data;
6848 : }
6849 :
6850 : /* Create the node partitions. */
6851 :
6852 : void
6853 709983 : vect_optimize_slp_pass::create_partitions ()
6854 : {
6855 : /* Calculate a postorder of the graph, ignoring edges that correspond
6856 : to natural latch edges in the cfg. Reading the vector from the end
6857 : to the beginning gives the reverse postorder. */
6858 709983 : auto_vec<int> initial_rpo;
6859 1419966 : graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
6860 : false, NULL, skip_cfg_latch_edges);
6861 2129949 : gcc_assert (initial_rpo.length () == m_vertices.length ());
6862 :
6863 : /* Calculate the strongly connected components of the graph. */
6864 709983 : auto_vec<int> scc_grouping;
6865 709983 : unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
6866 :
6867 : /* Create a new index order in which all nodes from the same SCC are
6868 : consecutive. Use scc_pos to record the index of the first node in
6869 : each SCC. */
6870 709983 : auto_vec<unsigned int> scc_pos (num_sccs);
6871 709983 : int last_component = -1;
6872 709983 : unsigned int node_count = 0;
6873 7354328 : for (unsigned int node_i : scc_grouping)
6874 : {
6875 5224379 : if (last_component != m_slpg->vertices[node_i].component)
6876 : {
6877 5093849 : last_component = m_slpg->vertices[node_i].component;
6878 10187698 : gcc_assert (last_component == int (scc_pos.length ()));
6879 5093849 : scc_pos.quick_push (node_count);
6880 : }
6881 5224379 : node_count += 1;
6882 : }
6883 1419966 : gcc_assert (node_count == initial_rpo.length ()
6884 : && last_component + 1 == int (num_sccs));
6885 :
6886 : /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
6887 : inside each SCC following the RPO we calculated above. The fact that
6888 : we ignored natural latch edges when calculating the RPO should ensure
6889 : that, for natural loop nests:
6890 :
6891 : - the first node that we encounter in a cfg loop is the loop header phi
6892 : - the loop header phis are in dominance order
6893 :
6894 : Arranging for this is an optimization (see below) rather than a
6895 : correctness issue. Unnatural loops with a tangled mess of backedges
6896 : will still work correctly, but might give poorer results.
6897 :
6898 : Also update scc_pos so that it gives 1 + the index of the last node
6899 : in the SCC. */
6900 709983 : m_partitioned_nodes.safe_grow (node_count);
6901 6644345 : for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
6902 : {
6903 5224379 : unsigned int node_i = initial_rpo[old_i];
6904 5224379 : unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
6905 5224379 : m_partitioned_nodes[new_i] = node_i;
6906 : }
6907 :
6908 : /* When optimizing for speed, partition each SCC based on the containing
6909 : cfg loop. The order we constructed above should ensure that, for natural
6910 : cfg loops, we'll create sub-SCC partitions for outer loops before
6911 : the corresponding sub-SCC partitions for inner loops. Similarly,
6912 : when one sibling loop A dominates another sibling loop B, we should
6913 : create a sub-SCC partition for A before a sub-SCC partition for B.
6914 :
6915 : As above, nothing depends for correctness on whether this achieves
6916 : a natural nesting, but we should get better results when it does. */
6917 1419966 : m_partitions.reserve (m_vertices.length ());
6918 709983 : unsigned int next_partition_i = 0;
6919 709983 : hash_map<struct loop *, int> loop_partitions;
6920 709983 : unsigned int rpo_begin = 0;
6921 709983 : unsigned int num_partitioned_nodes = 0;
6922 7223798 : for (unsigned int rpo_end : scc_pos)
6923 : {
6924 5093849 : loop_partitions.empty ();
6925 : unsigned int partition_i = next_partition_i;
6926 10318228 : for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
6927 : {
6928 : /* Handle externals and constants optimistically throughout.
6929 : But treat existing vectors as fixed since we do not handle
6930 : permuting them. */
6931 5224379 : unsigned int node_i = m_partitioned_nodes[rpo_i];
6932 5224379 : auto &vertex = m_vertices[node_i];
6933 5224379 : if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
6934 529607 : && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
6935 5227590 : || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
6936 1562552 : vertex.partition = -1;
6937 : else
6938 : {
6939 3661827 : bool existed;
6940 3661827 : if (m_optimize_size)
6941 27464 : existed = next_partition_i > partition_i;
6942 : else
6943 : {
6944 3634363 : struct loop *loop = containing_loop (vertex.node);
6945 3634363 : auto &entry = loop_partitions.get_or_insert (loop, &existed);
6946 3634363 : if (!existed)
6947 3504990 : entry = next_partition_i;
6948 3634363 : partition_i = entry;
6949 : }
6950 3661827 : if (!existed)
6951 : {
6952 3532362 : m_partitions.quick_push (slpg_partition_info ());
6953 3532362 : next_partition_i += 1;
6954 : }
6955 3661827 : vertex.partition = partition_i;
6956 3661827 : num_partitioned_nodes += 1;
6957 3661827 : m_partitions[partition_i].node_end += 1;
6958 : }
6959 : }
6960 5093849 : rpo_begin = rpo_end;
6961 : }
6962 :
6963 : /* Assign ranges of consecutive node indices to each partition,
6964 : in partition order. Start with node_end being the same as
6965 : node_begin so that the next loop can use it as a counter. */
6966 709983 : unsigned int node_begin = 0;
6967 5662311 : for (auto &partition : m_partitions)
6968 : {
6969 3532362 : partition.node_begin = node_begin;
6970 3532362 : node_begin += partition.node_end;
6971 3532362 : partition.node_end = partition.node_begin;
6972 : }
6973 709983 : gcc_assert (node_begin == num_partitioned_nodes);
6974 :
6975 : /* Finally build the list of nodes in partition order. */
6976 709983 : m_partitioned_nodes.truncate (num_partitioned_nodes);
6977 5934362 : for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
6978 : {
6979 5224379 : int partition_i = m_vertices[node_i].partition;
6980 5224379 : if (partition_i >= 0)
6981 : {
6982 3661827 : unsigned int order_i = m_partitions[partition_i].node_end++;
6983 3661827 : m_partitioned_nodes[order_i] = node_i;
6984 : }
6985 : }
6986 709983 : }
6987 :
6988 : /* Look for edges from earlier partitions into node NODE_I and edges from
6989 : node NODE_I into later partitions. Call:
6990 :
6991 : FN (ud, other_node_i)
6992 :
6993 : for each such use-to-def edge ud, where other_node_i is the node at the
6994 : other end of the edge. */
6995 :
6996 : template<typename T>
6997 : void
6998 4130791 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
6999 : {
7000 4130791 : int partition_i = m_vertices[node_i].partition;
7001 4130791 : for (graph_edge *pred = m_slpg->vertices[node_i].pred;
7002 7148940 : pred; pred = pred->pred_next)
7003 : {
7004 3018149 : int src_partition_i = m_vertices[pred->src].partition;
7005 3018149 : if (src_partition_i >= 0 && src_partition_i != partition_i)
7006 2688251 : fn (pred, pred->src);
7007 : }
7008 4130791 : for (graph_edge *succ = m_slpg->vertices[node_i].succ;
7009 8889986 : succ; succ = succ->succ_next)
7010 : {
7011 4759195 : int dest_partition_i = m_vertices[succ->dest].partition;
7012 4759195 : if (dest_partition_i >= 0 && dest_partition_i != partition_i)
7013 2724916 : fn (succ, succ->dest);
7014 : }
7015 4130791 : }
7016 :
7017 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7018 : that NODE would operate on. This test is independent of NODE's actual
7019 : operation. */
7020 :
7021 : bool
7022 1804549 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
7023 : unsigned int layout_i)
7024 : {
7025 1804549 : if (layout_i == 0)
7026 : return true;
7027 :
7028 1055560 : if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
7029 18439 : return false;
7030 :
7031 : return true;
7032 : }
7033 :
7034 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7035 : that NODE would operate on for each NODE in PARTITION.
7036 : This test is independent of NODE's actual operations. */
7037 :
7038 : bool
7039 23836 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
7040 : &partition,
7041 : unsigned int layout_i)
7042 : {
7043 48102 : for (unsigned int order_i = partition.node_begin;
7044 48102 : order_i < partition.node_end; ++order_i)
7045 : {
7046 24370 : unsigned int node_i = m_partitioned_nodes[order_i];
7047 24370 : auto &vertex = m_vertices[node_i];
7048 :
7049 : /* The layout is incompatible if it is individually incompatible
7050 : with any node in the partition. */
7051 24370 : if (!is_compatible_layout (vertex.node, layout_i))
7052 : return false;
7053 : }
7054 : return true;
7055 : }
7056 :
7057 : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
7058 : to layout TO_LAYOUT_I for a node like NODE. Return -1 if either of the
7059 : layouts is incompatible with NODE or if the change is not possible for
7060 : some other reason.
7061 :
7062 : The properties taken from NODE include the number of lanes and the
7063 : vector type. The actual operation doesn't matter. */
7064 :
7065 : int
7066 762302 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
7067 : unsigned int from_layout_i,
7068 : unsigned int to_layout_i)
7069 : {
7070 762302 : if (!is_compatible_layout (node, from_layout_i)
7071 762302 : || !is_compatible_layout (node, to_layout_i))
7072 623 : return -1;
7073 :
7074 761679 : if (from_layout_i == to_layout_i)
7075 : return 0;
7076 :
7077 317012 : auto_vec<slp_tree, 1> children (1);
7078 317012 : children.quick_push (node);
7079 317012 : auto_lane_permutation_t perm (SLP_TREE_LANES (node));
7080 317012 : if (from_layout_i > 0)
7081 890641 : for (unsigned int i : m_perms[from_layout_i])
7082 394375 : perm.quick_push ({ 0, i });
7083 : else
7084 496511 : for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
7085 344921 : perm.quick_push ({ 0, i });
7086 317012 : if (to_layout_i > 0)
7087 152333 : vect_slp_permute (m_perms[to_layout_i], perm, true);
7088 317012 : auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
7089 : children, false);
7090 317012 : if (count >= 0)
7091 311253 : return MAX (count, 1);
7092 :
7093 : /* ??? In principle we could try changing via layout 0, giving two
7094 : layout changes rather than 1. Doing that would require
7095 : corresponding support in get_result_with_layout. */
7096 : return -1;
7097 317012 : }
7098 :
7099 : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I. */
7100 :
7101 : inline slpg_partition_layout_costs &
7102 1136746 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
7103 : unsigned int layout_i)
7104 : {
7105 2273492 : return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
7106 : }
7107 :
7108 : /* Change PERM in one of two ways:
7109 :
7110 : - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
7111 : chosen for child I of NODE.
7112 :
7113 : - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
7114 :
7115 : In both cases, arrange for the output to have layout OUT_LAYOUT_I */
7116 :
7117 : void
7118 39085 : vect_optimize_slp_pass::
7119 : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
7120 : int in_layout_i, unsigned int out_layout_i)
7121 : {
7122 229327 : for (auto &entry : perm)
7123 : {
7124 112072 : int this_in_layout_i = in_layout_i;
7125 112072 : if (this_in_layout_i < 0)
7126 : {
7127 76473 : slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
7128 76473 : unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
7129 76473 : if (in_partition_i == -1u)
7130 329 : continue;
7131 76144 : this_in_layout_i = m_partitions[in_partition_i].layout;
7132 : }
7133 111743 : if (this_in_layout_i > 0)
7134 25087 : entry.second = m_perms[this_in_layout_i][entry.second];
7135 : }
7136 39085 : if (out_layout_i > 0)
7137 11305 : vect_slp_permute (m_perms[out_layout_i], perm, true);
7138 39085 : }
7139 :
7140 : /* Check whether the target allows NODE to be rearranged so that the node's
7141 : output has layout OUT_LAYOUT_I. Return the cost of the change if so,
7142 : in the same arbitrary units as for change_layout_cost. Return -1 otherwise.
7143 :
7144 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
7145 : NODE can adapt to the layout changes that have (perhaps provisionally)
7146 : been chosen for NODE's children, so that no extra permutations are
7147 : needed on either the input or the output of NODE.
7148 :
7149 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
7150 : that all inputs will be forced into layout IN_LAYOUT_I beforehand.
7151 :
7152 : IN_LAYOUT_I has no meaning for other types of node.
7153 :
7154 : Keeping the node as-is is always valid. If the target doesn't appear
7155 : to support the node as-is, but might realistically support other layouts,
7156 : then layout 0 instead has the cost of a worst-case permutation. On the
7157 : one hand, this ensures that every node has at least one valid layout,
7158 : avoiding what would otherwise be an awkward special case. On the other,
7159 : it still encourages the pass to change an invalid pre-existing layout
7160 : choice into a valid one. */
7161 :
7162 : int
7163 251321 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
7164 : unsigned int out_layout_i)
7165 : {
7166 251321 : const int fallback_cost = 1;
7167 :
7168 251321 : if (SLP_TREE_PERMUTE_P (node))
7169 : {
7170 32398 : auto_lane_permutation_t tmp_perm;
7171 32398 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7172 :
7173 : /* Check that the child nodes support the chosen layout. Checking
7174 : the first child is enough, since any second child would have the
7175 : same shape. */
7176 32398 : auto first_child = SLP_TREE_CHILDREN (node)[0];
7177 32398 : if (in_layout_i > 0
7178 32398 : && !is_compatible_layout (first_child, in_layout_i))
7179 : return -1;
7180 :
7181 31780 : change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
7182 63560 : int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
7183 : node, tmp_perm,
7184 31780 : SLP_TREE_CHILDREN (node),
7185 : false);
7186 31780 : if (count < 0)
7187 : {
7188 2593 : if (in_layout_i == 0 && out_layout_i == 0)
7189 : {
7190 : /* Use the fallback cost if the node could in principle support
7191 : some nonzero layout for both the inputs and the outputs.
7192 : Otherwise assume that the node will be rejected later
7193 : and rebuilt from scalars. */
7194 641 : if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
7195 : return fallback_cost;
7196 319 : return 0;
7197 : }
7198 : return -1;
7199 : }
7200 :
7201 : /* We currently have no way of telling whether the new layout is cheaper
7202 : or more expensive than the old one. But at least in principle,
7203 : it should be worth making zero permutations (whole-vector shuffles)
7204 : cheaper than real permutations, in case the pass is able to remove
7205 : the latter. */
7206 29187 : return count == 0 ? 0 : 1;
7207 32398 : }
7208 :
7209 218923 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
7210 218923 : if (rep
7211 216973 : && STMT_VINFO_DATA_REF (rep)
7212 71331 : && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
7213 266509 : && SLP_TREE_LOAD_PERMUTATION (node).exists ())
7214 : {
7215 39334 : auto_load_permutation_t tmp_perm;
7216 39334 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7217 39334 : if (out_layout_i > 0)
7218 15080 : vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
7219 :
7220 39334 : poly_uint64 vf = 1;
7221 39334 : if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
7222 12152 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7223 39334 : unsigned int n_perms;
7224 39334 : if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
7225 : nullptr, vf, true, false, &n_perms))
7226 : {
7227 2367 : auto rep = SLP_TREE_REPRESENTATIVE (node);
7228 2367 : if (out_layout_i == 0)
7229 : {
7230 : /* Use the fallback cost if the load is an N-to-N permutation.
7231 : Otherwise assume that the node will be rejected later
7232 : and rebuilt from scalars. */
7233 1734 : if (STMT_VINFO_GROUPED_ACCESS (rep)
7234 3468 : && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
7235 1734 : == SLP_TREE_LANES (node)))
7236 684 : return fallback_cost;
7237 : return 0;
7238 : }
7239 : return -1;
7240 : }
7241 :
7242 : /* See the comment above the corresponding VEC_PERM_EXPR handling. */
7243 36967 : return n_perms == 0 ? 0 : 1;
7244 39334 : }
7245 :
7246 : return 0;
7247 : }
7248 :
7249 : /* Decide which element layouts we should consider using. Calculate the
7250 : weights associated with inserting layout changes on partition edges.
7251 : Also mark partitions that cannot change layout, by setting their
7252 : layout to zero. */
7253 :
7254 : void
7255 709983 : vect_optimize_slp_pass::start_choosing_layouts ()
7256 : {
7257 : /* Used to assign unique permutation indices. */
7258 709983 : using perm_hash = unbounded_hashmap_traits<
7259 : vec_free_hash_base<int_hash_base<unsigned>>,
7260 : int_hash<int, -1, -2>
7261 : >;
7262 709983 : hash_map<vec<unsigned>, int, perm_hash> layout_ids;
7263 :
7264 : /* Layout 0 is "no change". */
7265 709983 : m_perms.safe_push (vNULL);
7266 :
7267 : /* Create layouts from existing permutations. */
7268 709983 : auto_load_permutation_t tmp_perm;
7269 5791776 : for (unsigned int node_i : m_partitioned_nodes)
7270 : {
7271 : /* Leafs also double as entries to the reverse graph. Allow the
7272 : layout of those to be changed. */
7273 3661827 : auto &vertex = m_vertices[node_i];
7274 3661827 : auto &partition = m_partitions[vertex.partition];
7275 3661827 : if (!m_slpg->vertices[node_i].succ)
7276 932800 : partition.layout = 0;
7277 :
7278 : /* Loads and VEC_PERM_EXPRs are the only things generating permutes. */
7279 3661827 : slp_tree node = vertex.node;
7280 3661827 : stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
7281 3661827 : slp_tree child;
7282 3661827 : unsigned HOST_WIDE_INT imin, imax = 0;
7283 3661827 : bool any_permute = false;
7284 3661827 : tmp_perm.truncate (0);
7285 3661827 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
7286 : {
7287 : /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
7288 : unpermuted, record a layout that reverses this permutation.
7289 :
7290 : We would need more work to cope with loads that are internally
7291 : permuted and also have inputs (such as masks for
7292 : IFN_MASK_LOADs). */
7293 628274 : gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
7294 628274 : if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
7295 : {
7296 443344 : partition.layout = -1;
7297 3640804 : continue;
7298 : }
7299 184930 : dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
7300 184930 : imin = DR_GROUP_SIZE (dr_stmt) + 1;
7301 184930 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7302 : }
7303 5946795 : else if (SLP_TREE_PERMUTE_P (node)
7304 139880 : && SLP_TREE_CHILDREN (node).length () == 1
7305 120311 : && (child = SLP_TREE_CHILDREN (node)[0])
7306 3153864 : && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
7307 120311 : .is_constant (&imin)))
7308 : {
7309 : /* If the child has the same vector size as this node,
7310 : reversing the permutation can make the permutation a no-op.
7311 : In other cases it can change a true permutation into a
7312 : full-vector extract. */
7313 120311 : tmp_perm.reserve (SLP_TREE_LANES (node));
7314 325420 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7315 205109 : tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
7316 : }
7317 : else
7318 2913242 : continue;
7319 :
7320 818694 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7321 : {
7322 513453 : unsigned idx = tmp_perm[j];
7323 513453 : imin = MIN (imin, idx);
7324 513453 : imax = MAX (imax, idx);
7325 513453 : if (idx - tmp_perm[0] != j)
7326 160613 : any_permute = true;
7327 : }
7328 : /* If the span doesn't match we'd disrupt VF computation, avoid
7329 : that for now. */
7330 305241 : if (imax - imin + 1 != SLP_TREE_LANES (node))
7331 91478 : continue;
7332 : /* If there's no permute no need to split one out. In this case
7333 : we can consider turning a load into a permuted load, if that
7334 : turns out to be cheaper than alternatives. */
7335 213763 : if (!any_permute)
7336 : {
7337 192585 : partition.layout = -1;
7338 192585 : continue;
7339 : }
7340 :
7341 : /* For now only handle true permutes, like
7342 : vect_attempt_slp_rearrange_stmts did. This allows us to be lazy
7343 : when permuting constants and invariants keeping the permute
7344 : bijective. */
7345 21178 : auto_sbitmap load_index (SLP_TREE_LANES (node));
7346 21178 : bitmap_clear (load_index);
7347 84622 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7348 63444 : bitmap_set_bit (load_index, tmp_perm[j] - imin);
7349 : unsigned j;
7350 83779 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
7351 62756 : if (!bitmap_bit_p (load_index, j))
7352 : break;
7353 21178 : if (j != SLP_TREE_LANES (node))
7354 155 : continue;
7355 :
7356 21023 : vec<unsigned> perm = vNULL;
7357 21023 : perm.safe_grow (SLP_TREE_LANES (node), true);
7358 83500 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7359 62477 : perm[j] = tmp_perm[j] - imin;
7360 :
7361 42046 : if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
7362 : {
7363 : /* Continue to use existing layouts, but don't add any more. */
7364 0 : int *entry = layout_ids.get (perm);
7365 0 : partition.layout = entry ? *entry : 0;
7366 0 : perm.release ();
7367 : }
7368 : else
7369 : {
7370 21023 : bool existed;
7371 21023 : int &layout_i = layout_ids.get_or_insert (perm, &existed);
7372 21023 : if (existed)
7373 7920 : perm.release ();
7374 : else
7375 : {
7376 13103 : layout_i = m_perms.length ();
7377 13103 : m_perms.safe_push (perm);
7378 : }
7379 21023 : partition.layout = layout_i;
7380 : }
7381 21178 : }
7382 :
7383 : /* Initially assume that every layout is possible and has zero cost
7384 : in every partition. */
7385 709983 : m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
7386 1419966 : * m_perms.length ());
7387 :
7388 : /* We have to mark outgoing permutations facing non-associating-reduction
7389 : graph entries that are not represented as to be materialized.
7390 : slp_inst_kind_bb_reduc currently only covers associatable reductions. */
7391 3655310 : for (slp_instance instance : m_vinfo->slp_instances)
7392 1525361 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
7393 : {
7394 6900 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7395 6900 : m_partitions[m_vertices[node_i].partition].layout = 0;
7396 : }
7397 1518461 : else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
7398 : {
7399 2306 : stmt_vec_info stmt_info
7400 2306 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
7401 2306 : vect_reduc_info reduc_info
7402 2306 : = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
7403 : SLP_INSTANCE_TREE (instance));
7404 2306 : if (needs_fold_left_reduction_p (TREE_TYPE
7405 : (gimple_get_lhs (stmt_info->stmt)),
7406 : VECT_REDUC_INFO_CODE (reduc_info)))
7407 : {
7408 100 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7409 100 : m_partitions[m_vertices[node_i].partition].layout = 0;
7410 : }
7411 : }
7412 :
7413 : /* Check which layouts each node and partition can handle. Calculate the
7414 : weights associated with inserting layout changes on edges. */
7415 5791776 : for (unsigned int node_i : m_partitioned_nodes)
7416 : {
7417 3661827 : auto &vertex = m_vertices[node_i];
7418 3661827 : auto &partition = m_partitions[vertex.partition];
7419 3661827 : slp_tree node = vertex.node;
7420 :
7421 3661827 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
7422 : {
7423 3655518 : vertex.weight = vect_slp_node_weight (node);
7424 :
7425 : /* We do not handle stores with a permutation, so all
7426 : incoming permutations must have been materialized.
7427 :
7428 : We also don't handle masked grouped loads, which lack a
7429 : permutation vector. In this case the memory locations
7430 : form an implicit second input to the loads, on top of the
7431 : explicit mask input, and the memory input's layout cannot
7432 : be changed.
7433 :
7434 : On the other hand, we do support permuting gather loads and
7435 : masked gather loads, where each scalar load is independent
7436 : of the others. This can be useful if the address/index input
7437 : benefits from permutation. */
7438 3655518 : if (STMT_VINFO_DATA_REF (rep)
7439 1812684 : && STMT_VINFO_GROUPED_ACCESS (rep)
7440 4773243 : && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
7441 932795 : partition.layout = 0;
7442 :
7443 : /* We cannot change the layout of an operation that is
7444 : not independent on lanes. Note this is an explicit
7445 : negative list since that's much shorter than the respective
7446 : positive one but it's critical to keep maintaining it. */
7447 3655518 : if (is_gimple_call (STMT_VINFO_STMT (rep)))
7448 33483 : switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
7449 : {
7450 1092 : case CFN_COMPLEX_ADD_ROT90:
7451 1092 : case CFN_COMPLEX_ADD_ROT270:
7452 1092 : case CFN_COMPLEX_MUL:
7453 1092 : case CFN_COMPLEX_MUL_CONJ:
7454 1092 : case CFN_VEC_ADDSUB:
7455 1092 : case CFN_VEC_FMADDSUB:
7456 1092 : case CFN_VEC_FMSUBADD:
7457 1092 : partition.layout = 0;
7458 : default:;
7459 : }
7460 : }
7461 :
7462 8281475 : auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
7463 : {
7464 4619648 : auto &other_vertex = m_vertices[other_node_i];
7465 :
7466 : /* Count the number of edges from earlier partitions and the number
7467 : of edges to later partitions. */
7468 4619648 : if (other_vertex.partition < vertex.partition)
7469 2309824 : partition.in_degree += 1;
7470 : else
7471 2309824 : partition.out_degree += 1;
7472 :
7473 : /* If the current node uses the result of OTHER_NODE_I, accumulate
7474 : the effects of that. */
7475 4619648 : if (ud->src == int (node_i))
7476 : {
7477 2309824 : other_vertex.out_weight += vertex.weight;
7478 2309824 : other_vertex.out_degree += 1;
7479 : }
7480 8281475 : };
7481 3661827 : for_each_partition_edge (node_i, process_edge);
7482 : }
7483 709983 : }
7484 :
7485 : /* Return the incoming costs for node NODE_I, assuming that each input keeps
7486 : its current (provisional) choice of layout. The inputs do not necessarily
7487 : have the same layout as each other. */
7488 :
7489 : slpg_layout_cost
7490 4457 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
7491 : {
7492 4457 : auto &vertex = m_vertices[node_i];
7493 4457 : slpg_layout_cost cost;
7494 14547 : auto add_cost = [&](graph_edge *, unsigned int other_node_i)
7495 : {
7496 10090 : auto &other_vertex = m_vertices[other_node_i];
7497 10090 : if (other_vertex.partition < vertex.partition)
7498 : {
7499 6887 : auto &other_partition = m_partitions[other_vertex.partition];
7500 13774 : auto &other_costs = partition_layout_costs (other_vertex.partition,
7501 6887 : other_partition.layout);
7502 6887 : slpg_layout_cost this_cost = other_costs.in_cost;
7503 6887 : this_cost.add_serial_cost (other_costs.internal_cost);
7504 6887 : this_cost.split (other_partition.out_degree);
7505 6887 : cost.add_parallel_cost (this_cost);
7506 : }
7507 14547 : };
7508 4457 : for_each_partition_edge (node_i, add_cost);
7509 4457 : return cost;
7510 : }
7511 :
7512 : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
7513 : and layout LAYOUT2_I on cross-partition use-to-def edge UD. Return
7514 : slpg_layout_cost::impossible () if the change isn't possible. */
7515 :
7516 : slpg_layout_cost
7517 762302 : vect_optimize_slp_pass::
7518 : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
7519 : unsigned int layout2_i)
7520 : {
7521 762302 : auto &def_vertex = m_vertices[ud->dest];
7522 762302 : auto &use_vertex = m_vertices[ud->src];
7523 762302 : auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
7524 762302 : auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
7525 762302 : auto factor = change_layout_cost (def_vertex.node, def_layout_i,
7526 : use_layout_i);
7527 762302 : if (factor < 0)
7528 6382 : return slpg_layout_cost::impossible ();
7529 :
7530 : /* We have a choice of putting the layout change at the site of the
7531 : definition or at the site of the use. Prefer the former when
7532 : optimizing for size or when the execution frequency of the
7533 : definition is no greater than the combined execution frequencies of
7534 : the uses. When putting the layout change at the site of the definition,
7535 : divvy up the cost among all consumers. */
7536 755920 : if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
7537 : {
7538 733238 : slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
7539 733238 : cost.split (def_vertex.out_degree);
7540 733238 : return cost;
7541 : }
7542 22682 : return { use_vertex.weight * factor, m_optimize_size };
7543 : }
7544 :
7545 : /* UD represents a use-def link between FROM_NODE_I and a node in a later
7546 : partition; FROM_NODE_I could be the definition node or the use node.
7547 : The node at the other end of the link wants to use layout TO_LAYOUT_I.
7548 : Return the cost of any necessary fix-ups on edge UD, or return
7549 : slpg_layout_cost::impossible () if the change isn't possible.
7550 :
7551 : At this point, FROM_NODE_I's partition has chosen the cheapest
7552 : layout based on the information available so far, but this choice
7553 : is only provisional. */
7554 :
7555 : slpg_layout_cost
7556 203388 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
7557 : unsigned int to_layout_i)
7558 : {
7559 203388 : auto &from_vertex = m_vertices[from_node_i];
7560 203388 : unsigned int from_partition_i = from_vertex.partition;
7561 203388 : slpg_partition_info &from_partition = m_partitions[from_partition_i];
7562 203388 : gcc_assert (from_partition.layout >= 0);
7563 :
7564 : /* First calculate the cost on the assumption that FROM_PARTITION sticks
7565 : with its current layout preference. */
7566 203388 : slpg_layout_cost cost = slpg_layout_cost::impossible ();
7567 203388 : auto edge_cost = edge_layout_cost (ud, from_node_i,
7568 203388 : from_partition.layout, to_layout_i);
7569 203388 : if (edge_cost.is_possible ())
7570 : {
7571 400058 : auto &from_costs = partition_layout_costs (from_partition_i,
7572 200029 : from_partition.layout);
7573 200029 : cost = from_costs.in_cost;
7574 200029 : cost.add_serial_cost (from_costs.internal_cost);
7575 200029 : cost.split (from_partition.out_degree);
7576 200029 : cost.add_serial_cost (edge_cost);
7577 : }
7578 3359 : else if (from_partition.layout == 0)
7579 : /* We must allow the source partition to have layout 0 as a fallback,
7580 : in case all other options turn out to be impossible. */
7581 3359 : return cost;
7582 :
7583 : /* Take the minimum of that cost and the cost that applies if
7584 : FROM_PARTITION instead switches to TO_LAYOUT_I. */
7585 200029 : auto &direct_layout_costs = partition_layout_costs (from_partition_i,
7586 : to_layout_i);
7587 200029 : if (direct_layout_costs.is_possible ())
7588 : {
7589 176128 : slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
7590 176128 : direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
7591 176128 : direct_cost.split (from_partition.out_degree);
7592 176128 : if (!cost.is_possible ()
7593 176128 : || direct_cost.is_better_than (cost, m_optimize_size))
7594 40187 : cost = direct_cost;
7595 : }
7596 :
7597 200029 : return cost;
7598 : }
7599 :
7600 : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
7601 : partition; TO_NODE_I could be the definition node or the use node.
7602 : The node at the other end of the link wants to use layout FROM_LAYOUT_I;
7603 : return the cost of any necessary fix-ups on edge UD, or
7604 : slpg_layout_cost::impossible () if the choice cannot be made.
7605 :
7606 : At this point, TO_NODE_I's partition has a fixed choice of layout. */
7607 :
7608 : slpg_layout_cost
7609 182978 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
7610 : unsigned int from_layout_i)
7611 : {
7612 182978 : auto &to_vertex = m_vertices[to_node_i];
7613 182978 : unsigned int to_partition_i = to_vertex.partition;
7614 182978 : slpg_partition_info &to_partition = m_partitions[to_partition_i];
7615 182978 : gcc_assert (to_partition.layout >= 0);
7616 :
7617 : /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
7618 : adjusted for this input having layout FROM_LAYOUT_I. Assume that
7619 : any other inputs keep their current choice of layout. */
7620 182978 : auto &to_costs = partition_layout_costs (to_partition_i,
7621 : to_partition.layout);
7622 182978 : if (ud->src == int (to_node_i)
7623 182776 : && SLP_TREE_PERMUTE_P (to_vertex.node))
7624 : {
7625 11525 : auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
7626 11525 : auto old_layout = from_partition.layout;
7627 11525 : from_partition.layout = from_layout_i;
7628 23050 : int factor = internal_node_cost (to_vertex.node, -1,
7629 11525 : to_partition.layout);
7630 11525 : from_partition.layout = old_layout;
7631 11525 : if (factor >= 0)
7632 : {
7633 10621 : slpg_layout_cost cost = to_costs.out_cost;
7634 21242 : cost.add_serial_cost ({ to_vertex.weight * factor,
7635 10621 : m_optimize_size });
7636 10621 : cost.split (to_partition.in_degree);
7637 10621 : return cost;
7638 : }
7639 : }
7640 :
7641 : /* Compute the cost if we insert any necessary layout change on edge UD. */
7642 172357 : auto edge_cost = edge_layout_cost (ud, to_node_i,
7643 172357 : to_partition.layout, from_layout_i);
7644 172357 : if (edge_cost.is_possible ())
7645 : {
7646 172357 : slpg_layout_cost cost = to_costs.out_cost;
7647 172357 : cost.add_serial_cost (to_costs.internal_cost);
7648 172357 : cost.split (to_partition.in_degree);
7649 172357 : cost.add_serial_cost (edge_cost);
7650 172357 : return cost;
7651 : }
7652 :
7653 0 : return slpg_layout_cost::impossible ();
7654 : }
7655 :
7656 : /* Make a forward pass through the partitions, accumulating input costs.
7657 : Make a tentative (provisional) choice of layout for each partition,
7658 : ensuring that this choice still allows later partitions to keep
7659 : their original layout. */
7660 :
7661 : void
7662 6595 : vect_optimize_slp_pass::forward_pass ()
7663 : {
7664 135559 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
7665 : ++partition_i)
7666 : {
7667 128964 : auto &partition = m_partitions[partition_i];
7668 :
7669 : /* If the partition consists of a single VEC_PERM_EXPR, precompute
7670 : the incoming cost that would apply if every predecessor partition
7671 : keeps its current layout. This is used within the loop below. */
7672 128964 : slpg_layout_cost in_cost;
7673 128964 : slp_tree single_node = nullptr;
7674 128964 : if (partition.node_end == partition.node_begin + 1)
7675 : {
7676 122589 : unsigned int node_i = m_partitioned_nodes[partition.node_begin];
7677 122589 : single_node = m_vertices[node_i].node;
7678 122589 : if (SLP_TREE_PERMUTE_P (single_node))
7679 4457 : in_cost = total_in_cost (node_i);
7680 : }
7681 :
7682 : /* Go through the possible layouts. Decide which ones are valid
7683 : for this partition and record which of the valid layouts has
7684 : the lowest cost. */
7685 128964 : unsigned int min_layout_i = 0;
7686 128964 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7687 397289 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7688 : {
7689 268325 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7690 268325 : if (!layout_costs.is_possible ())
7691 67581 : continue;
7692 :
7693 : /* If the recorded layout is already 0 then the layout cannot
7694 : change. */
7695 268325 : if (partition.layout == 0 && layout_i != 0)
7696 : {
7697 45736 : layout_costs.mark_impossible ();
7698 45736 : continue;
7699 : }
7700 :
7701 222589 : bool is_possible = true;
7702 453047 : for (unsigned int order_i = partition.node_begin;
7703 453047 : order_i < partition.node_end; ++order_i)
7704 : {
7705 249128 : unsigned int node_i = m_partitioned_nodes[order_i];
7706 249128 : auto &vertex = m_vertices[node_i];
7707 :
7708 : /* Reject the layout if it is individually incompatible
7709 : with any node in the partition. */
7710 249128 : if (!is_compatible_layout (vertex.node, layout_i))
7711 : {
7712 17094 : is_possible = false;
7713 18670 : break;
7714 : }
7715 :
7716 622415 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7717 : {
7718 390381 : auto &other_vertex = m_vertices[other_node_i];
7719 390381 : if (other_vertex.partition < vertex.partition)
7720 : {
7721 : /* Accumulate the incoming costs from earlier
7722 : partitions, plus the cost of any layout changes
7723 : on UD itself. */
7724 203388 : auto cost = forward_cost (ud, other_node_i, layout_i);
7725 203388 : if (!cost.is_possible ())
7726 3359 : is_possible = false;
7727 : else
7728 200029 : layout_costs.in_cost.add_parallel_cost (cost);
7729 : }
7730 : else
7731 : /* Reject the layout if it would make layout 0 impossible
7732 : for later partitions. This amounts to testing that the
7733 : target supports reversing the layout change on edges
7734 : to later partitions.
7735 :
7736 : In principle, it might be possible to push a layout
7737 : change all the way down a graph, so that it never
7738 : needs to be reversed and so that the target doesn't
7739 : need to support the reverse operation. But it would
7740 : be awkward to bail out if we hit a partition that
7741 : does not support the new layout, especially since
7742 : we are not dealing with a lattice. */
7743 186993 : is_possible &= edge_layout_cost (ud, other_node_i, 0,
7744 186993 : layout_i).is_possible ();
7745 622415 : };
7746 232034 : for_each_partition_edge (node_i, add_cost);
7747 :
7748 : /* Accumulate the cost of using LAYOUT_I within NODE,
7749 : both for the inputs and the outputs. */
7750 232034 : int factor = internal_node_cost (vertex.node, layout_i,
7751 : layout_i);
7752 232034 : if (factor < 0)
7753 : {
7754 1576 : is_possible = false;
7755 1576 : break;
7756 : }
7757 230458 : else if (factor)
7758 36372 : layout_costs.internal_cost.add_serial_cost
7759 36372 : ({ vertex.weight * factor, m_optimize_size });
7760 : }
7761 222589 : if (!is_possible)
7762 : {
7763 21845 : layout_costs.mark_impossible ();
7764 21845 : continue;
7765 : }
7766 :
7767 : /* Combine the incoming and partition-internal costs. */
7768 200744 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7769 200744 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7770 :
7771 : /* If this partition consists of a single VEC_PERM_EXPR, see
7772 : if the VEC_PERM_EXPR can be changed to support output layout
7773 : LAYOUT_I while keeping all the provisional choices of input
7774 : layout. */
7775 200744 : if (single_node && SLP_TREE_PERMUTE_P (single_node))
7776 : {
7777 7762 : int factor = internal_node_cost (single_node, -1, layout_i);
7778 7762 : if (factor >= 0)
7779 : {
7780 7039 : auto weight = m_vertices[single_node->vertex].weight;
7781 7039 : slpg_layout_cost internal_cost
7782 7039 : = { weight * factor, m_optimize_size };
7783 :
7784 7039 : slpg_layout_cost alt_cost = in_cost;
7785 7039 : alt_cost.add_serial_cost (internal_cost);
7786 7039 : if (alt_cost.is_better_than (combined_cost, m_optimize_size))
7787 : {
7788 2650 : combined_cost = alt_cost;
7789 2650 : layout_costs.in_cost = in_cost;
7790 2650 : layout_costs.internal_cost = internal_cost;
7791 : }
7792 : }
7793 : }
7794 :
7795 : /* Record the layout with the lowest cost. Prefer layout 0 in
7796 : the event of a tie between it and another layout. */
7797 200744 : if (!min_layout_cost.is_possible ()
7798 71780 : || combined_cost.is_better_than (min_layout_cost,
7799 71780 : m_optimize_size))
7800 : {
7801 145977 : min_layout_i = layout_i;
7802 145977 : min_layout_cost = combined_cost;
7803 : }
7804 : }
7805 :
7806 : /* This loop's handling of earlier partitions should ensure that
7807 : choosing the original layout for the current partition is no
7808 : less valid than it was in the original graph, even with the
7809 : provisional layout choices for those earlier partitions. */
7810 128964 : gcc_assert (min_layout_cost.is_possible ());
7811 128964 : partition.layout = min_layout_i;
7812 : }
7813 6595 : }
7814 :
7815 : /* Make a backward pass through the partitions, accumulating output costs.
7816 : Make a final choice of layout for each partition. */
7817 :
7818 : void
7819 6595 : vect_optimize_slp_pass::backward_pass ()
7820 : {
7821 142154 : for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
7822 : {
7823 128964 : auto &partition = m_partitions[partition_i];
7824 :
7825 128964 : unsigned int min_layout_i = 0;
7826 128964 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7827 397289 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7828 : {
7829 268325 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7830 268325 : if (!layout_costs.is_possible ())
7831 67581 : continue;
7832 :
7833 : /* Accumulate the costs from successor partitions. */
7834 200744 : bool is_possible = true;
7835 427996 : for (unsigned int order_i = partition.node_begin;
7836 427996 : order_i < partition.node_end; ++order_i)
7837 : {
7838 227252 : unsigned int node_i = m_partitioned_nodes[order_i];
7839 227252 : auto &vertex = m_vertices[node_i];
7840 609794 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7841 : {
7842 382542 : auto &other_vertex = m_vertices[other_node_i];
7843 382542 : auto &other_partition = m_partitions[other_vertex.partition];
7844 382542 : if (other_vertex.partition > vertex.partition)
7845 : {
7846 : /* Accumulate the incoming costs from later
7847 : partitions, plus the cost of any layout changes
7848 : on UD itself. */
7849 182978 : auto cost = backward_cost (ud, other_node_i, layout_i);
7850 182978 : if (!cost.is_possible ())
7851 0 : is_possible = false;
7852 : else
7853 182978 : layout_costs.out_cost.add_parallel_cost (cost);
7854 : }
7855 : else
7856 : /* Make sure that earlier partitions can (if necessary
7857 : or beneficial) keep the layout that they chose in
7858 : the forward pass. This ensures that there is at
7859 : least one valid choice of layout. */
7860 199564 : is_possible &= edge_layout_cost (ud, other_node_i,
7861 199564 : other_partition.layout,
7862 199564 : layout_i).is_possible ();
7863 609794 : };
7864 227252 : for_each_partition_edge (node_i, add_cost);
7865 : }
7866 200744 : if (!is_possible)
7867 : {
7868 0 : layout_costs.mark_impossible ();
7869 0 : continue;
7870 : }
7871 :
7872 : /* Locally combine the costs from the forward and backward passes.
7873 : (This combined cost is not passed on, since that would lead
7874 : to double counting.) */
7875 200744 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7876 200744 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7877 200744 : combined_cost.add_serial_cost (layout_costs.out_cost);
7878 :
7879 : /* Record the layout with the lowest cost. Prefer layout 0 in
7880 : the event of a tie between it and another layout. */
7881 200744 : if (!min_layout_cost.is_possible ()
7882 71780 : || combined_cost.is_better_than (min_layout_cost,
7883 71780 : m_optimize_size))
7884 : {
7885 139559 : min_layout_i = layout_i;
7886 139559 : min_layout_cost = combined_cost;
7887 : }
7888 : }
7889 :
7890 128964 : gcc_assert (min_layout_cost.is_possible ());
7891 128964 : partition.layout = min_layout_i;
7892 : }
7893 6595 : }
7894 :
7895 : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
7896 : NODE already has the layout that was selected for its partition. */
7897 :
7898 : slp_tree
7899 177904 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
7900 : unsigned int to_layout_i)
7901 : {
7902 177904 : unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
7903 177904 : slp_tree result = m_node_layouts[result_i];
7904 177904 : if (result)
7905 : return result;
7906 :
7907 177052 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
7908 177052 : || (SLP_TREE_DEF_TYPE (node) == vect_external_def
7909 : /* We can't permute vector defs in place. */
7910 21728 : && SLP_TREE_VEC_DEFS (node).is_empty ()))
7911 : {
7912 : /* If the vector is uniform or unchanged, there's nothing to do. */
7913 45216 : if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
7914 : result = node;
7915 : else
7916 : {
7917 2700 : auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
7918 2700 : result = vect_create_new_slp_node (scalar_ops);
7919 2700 : vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
7920 : }
7921 : }
7922 : else
7923 : {
7924 131836 : unsigned int partition_i = m_vertices[node->vertex].partition;
7925 131836 : unsigned int from_layout_i = m_partitions[partition_i].layout;
7926 131836 : if (from_layout_i == to_layout_i)
7927 130763 : return node;
7928 :
7929 : /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
7930 : permutation instead of a serial one. Leave the new permutation
7931 : in TMP_PERM on success. */
7932 1073 : auto_lane_permutation_t tmp_perm;
7933 1073 : unsigned int num_inputs = 1;
7934 1073 : if (SLP_TREE_PERMUTE_P (node))
7935 : {
7936 7 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7937 7 : if (from_layout_i != 0)
7938 7 : vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
7939 7 : if (to_layout_i != 0)
7940 4 : vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
7941 7 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7942 : tmp_perm,
7943 7 : SLP_TREE_CHILDREN (node),
7944 : false) >= 0)
7945 7 : num_inputs = SLP_TREE_CHILDREN (node).length ();
7946 : else
7947 0 : tmp_perm.truncate (0);
7948 : }
7949 :
7950 1073 : if (dump_enabled_p ())
7951 : {
7952 70 : if (tmp_perm.length () > 0)
7953 6 : dump_printf_loc (MSG_NOTE, vect_location,
7954 : "duplicating permutation node %p with"
7955 : " layout %d\n",
7956 : (void *) node, to_layout_i);
7957 : else
7958 64 : dump_printf_loc (MSG_NOTE, vect_location,
7959 : "inserting permutation node in place of %p\n",
7960 : (void *) node);
7961 : }
7962 :
7963 1073 : unsigned int num_lanes = SLP_TREE_LANES (node);
7964 1073 : result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
7965 1073 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
7966 : {
7967 1068 : auto &stmts = SLP_TREE_SCALAR_STMTS (result);
7968 1068 : stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
7969 1068 : if (from_layout_i != 0)
7970 450 : vect_slp_permute (m_perms[from_layout_i], stmts, false);
7971 1068 : if (to_layout_i != 0)
7972 628 : vect_slp_permute (m_perms[to_layout_i], stmts, true);
7973 : }
7974 1073 : SLP_TREE_REPRESENTATIVE (result) = SLP_TREE_REPRESENTATIVE (node);
7975 1073 : SLP_TREE_LANES (result) = num_lanes;
7976 1073 : SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
7977 1073 : result->vertex = -1;
7978 :
7979 1073 : auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
7980 1073 : if (tmp_perm.length ())
7981 : {
7982 7 : lane_perm.safe_splice (tmp_perm);
7983 7 : SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
7984 : }
7985 : else
7986 : {
7987 1066 : lane_perm.create (num_lanes);
7988 3300 : for (unsigned j = 0; j < num_lanes; ++j)
7989 2234 : lane_perm.quick_push ({ 0, j });
7990 1066 : if (from_layout_i != 0)
7991 443 : vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
7992 1066 : if (to_layout_i != 0)
7993 629 : vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
7994 1066 : SLP_TREE_CHILDREN (result).safe_push (node);
7995 : }
7996 4296 : for (slp_tree child : SLP_TREE_CHILDREN (result))
7997 1077 : child->refcnt++;
7998 1073 : }
7999 46289 : m_node_layouts[result_i] = result;
8000 46289 : return result;
8001 : }
8002 :
8003 : /* Apply the chosen vector layouts to the SLP graph. */
8004 :
8005 : void
8006 12525 : vect_optimize_slp_pass::materialize ()
8007 : {
8008 : /* We no longer need the costs, so avoid having two O(N * P) arrays
8009 : live at the same time. */
8010 12525 : m_partition_layout_costs.release ();
8011 37575 : m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
8012 :
8013 25050 : auto_sbitmap fully_folded (m_vertices.length ());
8014 12525 : bitmap_clear (fully_folded);
8015 192889 : for (unsigned int node_i : m_partitioned_nodes)
8016 : {
8017 155314 : auto &vertex = m_vertices[node_i];
8018 155314 : slp_tree node = vertex.node;
8019 155314 : int layout_i = m_partitions[vertex.partition].layout;
8020 155314 : gcc_assert (layout_i >= 0);
8021 :
8022 : /* Rearrange the scalar statements to match the chosen layout. */
8023 155314 : if (layout_i > 0)
8024 22011 : vect_slp_permute (m_perms[layout_i],
8025 22011 : SLP_TREE_SCALAR_STMTS (node), true);
8026 :
8027 : /* Update load and lane permutations. */
8028 155314 : if (SLP_TREE_PERMUTE_P (node))
8029 : {
8030 : /* First try to absorb the input vector layouts. If that fails,
8031 : force the inputs to have layout LAYOUT_I too. We checked that
8032 : that was possible before deciding to use nonzero output layouts.
8033 : (Note that at this stage we don't really have any guarantee that
8034 : the target supports the original VEC_PERM_EXPR.) */
8035 6664 : auto &perm = SLP_TREE_LANE_PERMUTATION (node);
8036 6664 : auto_lane_permutation_t tmp_perm;
8037 6664 : tmp_perm.safe_splice (perm);
8038 6664 : change_vec_perm_layout (node, tmp_perm, -1, layout_i);
8039 6664 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
8040 : tmp_perm,
8041 6664 : SLP_TREE_CHILDREN (node),
8042 : false) >= 0)
8043 : {
8044 6023 : if (dump_enabled_p ()
8045 6943 : && !std::equal (tmp_perm.begin (), tmp_perm.end (),
8046 : perm.begin ()))
8047 58 : dump_printf_loc (MSG_NOTE, vect_location,
8048 : "absorbing input layouts into %p\n",
8049 : (void *) node);
8050 34136 : std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
8051 6023 : bitmap_set_bit (fully_folded, node_i);
8052 : }
8053 : else
8054 : {
8055 : /* Not MSG_MISSED because it would make no sense to users. */
8056 641 : if (dump_enabled_p ())
8057 46 : dump_printf_loc (MSG_NOTE, vect_location,
8058 : "failed to absorb input layouts into %p\n",
8059 : (void *) node);
8060 641 : change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
8061 : }
8062 6664 : }
8063 : else
8064 : {
8065 148650 : gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
8066 148650 : auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
8067 148650 : if (layout_i > 0)
8068 : /* ??? When we handle non-bijective permutes the idea
8069 : is that we can force the load-permutation to be
8070 : { min, min + 1, min + 2, ... max }. But then the
8071 : scalar defs might no longer match the lane content
8072 : which means wrong-code with live lane vectorization.
8073 : So we possibly have to have NULL entries for those. */
8074 21067 : vect_slp_permute (m_perms[layout_i], load_perm, true);
8075 : }
8076 : }
8077 :
8078 : /* Do this before any nodes disappear, since it involves a walk
8079 : over the leaves. */
8080 12525 : remove_redundant_permutations ();
8081 :
8082 : /* Replace each child with a correctly laid-out version. */
8083 192889 : for (unsigned int node_i : m_partitioned_nodes)
8084 : {
8085 : /* Skip nodes that have already been handled above. */
8086 155314 : if (bitmap_bit_p (fully_folded, node_i))
8087 6023 : continue;
8088 :
8089 149291 : auto &vertex = m_vertices[node_i];
8090 149291 : int in_layout_i = m_partitions[vertex.partition].layout;
8091 149291 : gcc_assert (in_layout_i >= 0);
8092 :
8093 : unsigned j;
8094 : slp_tree child;
8095 447216 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
8096 : {
8097 183921 : if (!child)
8098 6017 : continue;
8099 :
8100 177904 : slp_tree new_child = get_result_with_layout (child, in_layout_i);
8101 177904 : if (new_child != child)
8102 : {
8103 4342 : vect_free_slp_tree (child);
8104 4342 : SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
8105 4342 : new_child->refcnt += 1;
8106 : }
8107 : }
8108 : }
8109 12525 : }
8110 :
8111 : /* Elide load permutations that are not necessary. Such permutations might
8112 : be pre-existing, rather than created by the layout optimizations. */
8113 :
8114 : void
8115 709983 : vect_optimize_slp_pass::remove_redundant_permutations ()
8116 : {
8117 4717581 : for (unsigned int node_i : m_leafs)
8118 : {
8119 2587632 : slp_tree node = m_vertices[node_i].node;
8120 2587632 : if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
8121 1959358 : continue;
8122 :
8123 : /* In basic block vectorization we allow any subchain of an interleaving
8124 : chain.
8125 : FORNOW: not in loop SLP because of realignment complications. */
8126 628274 : if (is_a <bb_vec_info> (m_vinfo))
8127 : {
8128 188888 : bool subchain_p = true;
8129 : stmt_vec_info next_load_info = NULL;
8130 : stmt_vec_info load_info;
8131 : unsigned j;
8132 188888 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
8133 : {
8134 156477 : if (j != 0
8135 156477 : && (next_load_info != load_info
8136 69650 : || ! load_info
8137 69650 : || DR_GROUP_GAP (load_info) != 1))
8138 : {
8139 : subchain_p = false;
8140 : break;
8141 : }
8142 125306 : next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
8143 : }
8144 63582 : if (subchain_p)
8145 : {
8146 32411 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8147 32411 : continue;
8148 : }
8149 : }
8150 : else
8151 : {
8152 564692 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
8153 564692 : bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
8154 : /* When this isn't a grouped access we know it's single element
8155 : and contiguous. */
8156 564692 : if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
8157 : {
8158 443344 : if (!this_load_permuted
8159 443344 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8160 442583 : || SLP_TREE_LANES (node) == 1))
8161 442585 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8162 443344 : continue;
8163 : }
8164 121348 : stmt_vec_info first_stmt_info
8165 121348 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
8166 121852 : if (!this_load_permuted
8167 : /* The load requires permutation when unrolling exposes
8168 : a gap either because the group is larger than the SLP
8169 : group-size or because there is a gap between the groups. */
8170 121348 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8171 98850 : || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
8172 136 : && DR_GROUP_GAP (first_stmt_info) == 0)))
8173 : {
8174 504 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8175 504 : continue;
8176 : }
8177 : }
8178 : }
8179 709983 : }
8180 :
8181 : /* Print the partition graph and layout information to the dump file. */
8182 :
8183 : void
8184 679 : vect_optimize_slp_pass::dump ()
8185 : {
8186 679 : dump_printf_loc (MSG_NOTE, vect_location,
8187 : "SLP optimize permutations:\n");
8188 1371 : for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
8189 : {
8190 692 : dump_printf_loc (MSG_NOTE, vect_location, " %d: { ", layout_i);
8191 692 : const char *sep = "";
8192 5909 : for (unsigned int idx : m_perms[layout_i])
8193 : {
8194 3833 : dump_printf (MSG_NOTE, "%s%d", sep, idx);
8195 3833 : sep = ", ";
8196 : }
8197 692 : dump_printf (MSG_NOTE, " }\n");
8198 : }
8199 679 : dump_printf_loc (MSG_NOTE, vect_location,
8200 : "SLP optimize partitions:\n");
8201 5666 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
8202 : ++partition_i)
8203 : {
8204 4987 : auto &partition = m_partitions[partition_i];
8205 4987 : dump_printf_loc (MSG_NOTE, vect_location, " -------------\n");
8206 4987 : dump_printf_loc (MSG_NOTE, vect_location,
8207 : " partition %d (layout %d):\n",
8208 : partition_i, partition.layout);
8209 4987 : dump_printf_loc (MSG_NOTE, vect_location, " nodes:\n");
8210 10208 : for (unsigned int order_i = partition.node_begin;
8211 10208 : order_i < partition.node_end; ++order_i)
8212 : {
8213 5221 : auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
8214 10442 : dump_printf_loc (MSG_NOTE, vect_location, " - %p:\n",
8215 5221 : (void *) vertex.node);
8216 5221 : dump_printf_loc (MSG_NOTE, vect_location,
8217 : " weight: %f\n",
8218 : vertex.weight.to_double ());
8219 5221 : if (vertex.out_degree)
8220 4080 : dump_printf_loc (MSG_NOTE, vect_location,
8221 : " out weight: %f (degree %d)\n",
8222 : vertex.out_weight.to_double (),
8223 : vertex.out_degree);
8224 5221 : if (SLP_TREE_PERMUTE_P (vertex.node))
8225 506 : dump_printf_loc (MSG_NOTE, vect_location,
8226 : " op: VEC_PERM_EXPR\n");
8227 4715 : else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
8228 4697 : dump_printf_loc (MSG_NOTE, vect_location,
8229 : " op template: %G", rep->stmt);
8230 : }
8231 4987 : dump_printf_loc (MSG_NOTE, vect_location, " edges:\n");
8232 10208 : for (unsigned int order_i = partition.node_begin;
8233 10208 : order_i < partition.node_end; ++order_i)
8234 : {
8235 5221 : unsigned int node_i = m_partitioned_nodes[order_i];
8236 5221 : auto &vertex = m_vertices[node_i];
8237 15727 : auto print_edge = [&](graph_edge *, unsigned int other_node_i)
8238 : {
8239 10506 : auto &other_vertex = m_vertices[other_node_i];
8240 10506 : if (other_vertex.partition < vertex.partition)
8241 5253 : dump_printf_loc (MSG_NOTE, vect_location,
8242 : " - %p [%d] --> %p\n",
8243 5253 : (void *) other_vertex.node,
8244 : other_vertex.partition,
8245 5253 : (void *) vertex.node);
8246 : else
8247 5253 : dump_printf_loc (MSG_NOTE, vect_location,
8248 : " - %p --> [%d] %p\n",
8249 5253 : (void *) vertex.node,
8250 : other_vertex.partition,
8251 5253 : (void *) other_vertex.node);
8252 15727 : };
8253 5221 : for_each_partition_edge (node_i, print_edge);
8254 : }
8255 :
8256 15160 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
8257 : {
8258 10173 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
8259 10173 : if (layout_costs.is_possible ())
8260 : {
8261 8384 : dump_printf_loc (MSG_NOTE, vect_location,
8262 : " layout %d:%s\n", layout_i,
8263 8384 : partition.layout == int (layout_i)
8264 : ? " (*)" : "");
8265 8384 : slpg_layout_cost combined_cost = layout_costs.in_cost;
8266 8384 : combined_cost.add_serial_cost (layout_costs.internal_cost);
8267 8384 : combined_cost.add_serial_cost (layout_costs.out_cost);
8268 : #define TEMPLATE "{depth: %f, total: %f}"
8269 8384 : dump_printf_loc (MSG_NOTE, vect_location,
8270 : " " TEMPLATE "\n",
8271 : layout_costs.in_cost.depth.to_double (),
8272 : layout_costs.in_cost.total.to_double ());
8273 8384 : dump_printf_loc (MSG_NOTE, vect_location,
8274 : " + " TEMPLATE "\n",
8275 : layout_costs.internal_cost.depth.to_double (),
8276 : layout_costs.internal_cost.total.to_double ());
8277 8384 : dump_printf_loc (MSG_NOTE, vect_location,
8278 : " + " TEMPLATE "\n",
8279 : layout_costs.out_cost.depth.to_double (),
8280 : layout_costs.out_cost.total.to_double ());
8281 8384 : dump_printf_loc (MSG_NOTE, vect_location,
8282 : " = " TEMPLATE "\n",
8283 : combined_cost.depth.to_double (),
8284 : combined_cost.total.to_double ());
8285 : #undef TEMPLATE
8286 : }
8287 : else
8288 1789 : dump_printf_loc (MSG_NOTE, vect_location,
8289 : " layout %d: rejected\n", layout_i);
8290 : }
8291 : }
8292 679 : }
8293 :
8294 : /* Masked load lanes discovery. */
8295 :
8296 : void
8297 709983 : vect_optimize_slp_pass::decide_masked_load_lanes ()
8298 : {
8299 7355399 : for (auto v : m_vertices)
8300 : {
8301 5225450 : slp_tree node = v.node;
8302 5225450 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8303 3659682 : || SLP_TREE_PERMUTE_P (node))
8304 1706719 : continue;
8305 3518731 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
8306 1692229 : if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
8307 : /* The mask has to be uniform. */
8308 997933 : || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
8309 997855 : || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
8310 3518816 : || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
8311 : IFN_MASK_LOAD))
8312 3518698 : continue;
8313 33 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8314 66 : if (STMT_VINFO_STRIDED_P (stmt_info)
8315 33 : || compare_step_with_zero (m_vinfo, stmt_info) <= 0
8316 63 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
8317 30 : DR_GROUP_SIZE (stmt_info),
8318 : true) == IFN_LAST)
8319 33 : continue;
8320 :
8321 : /* Uniform masks need to be suitably represented. */
8322 0 : slp_tree mask = SLP_TREE_CHILDREN (node)[0];
8323 0 : if (!SLP_TREE_PERMUTE_P (mask)
8324 0 : || SLP_TREE_CHILDREN (mask).length () != 1)
8325 0 : continue;
8326 0 : bool match = true;
8327 0 : for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
8328 0 : if (perm.first != 0 || perm.second != 0)
8329 : {
8330 : match = false;
8331 : break;
8332 : }
8333 0 : if (!match)
8334 0 : continue;
8335 :
8336 : /* Now see if the consumer side matches. */
8337 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8338 0 : pred; pred = pred->pred_next)
8339 : {
8340 0 : slp_tree pred_node = m_vertices[pred->src].node;
8341 : /* All consumers should be a permute with a single outgoing lane. */
8342 0 : if (!SLP_TREE_PERMUTE_P (pred_node)
8343 0 : || SLP_TREE_LANES (pred_node) != 1)
8344 : {
8345 : match = false;
8346 : break;
8347 : }
8348 0 : gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
8349 : }
8350 0 : if (!match)
8351 0 : continue;
8352 : /* Now we can mark the nodes as to use load lanes. */
8353 0 : node->ldst_lanes = true;
8354 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8355 0 : pred; pred = pred->pred_next)
8356 0 : m_vertices[pred->src].node->ldst_lanes = true;
8357 : /* The catch is we have to massage the mask. We have arranged
8358 : analyzed uniform masks to be represented by a splat VEC_PERM
8359 : which we can now simply elide as we cannot easily re-do SLP
8360 : discovery here. */
8361 0 : slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
8362 0 : SLP_TREE_REF_COUNT (new_mask)++;
8363 0 : SLP_TREE_CHILDREN (node)[0] = new_mask;
8364 0 : vect_free_slp_tree (mask);
8365 : }
8366 709983 : }
8367 :
8368 : /* Perform legitimizing attempts. This is intended to improve the
8369 : situation when layout 0 is not valid which is a situation the cost
8370 : based propagation does not handle well.
8371 : Return true if further layout optimization is possible, false if
8372 : the layout configuration should be considered final. */
8373 :
8374 : bool
8375 12525 : vect_optimize_slp_pass::legitimize ()
8376 : {
8377 : /* Perform a very simple legitimizing attempt by attempting to choose
8378 : a single layout for all partitions that will make all permutations
8379 : a noop. That should also be the optimal layout choice in case
8380 : layout zero is legitimate.
8381 : ??? Disconnected components of the SLP graph could have distinct
8382 : single layouts. */
8383 12525 : int single_layout_i = -1;
8384 12525 : unsigned deferred_up_to = -1U;
8385 39961 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8386 : ++partition_i)
8387 : {
8388 34016 : auto &partition = m_partitions[partition_i];
8389 34016 : if (single_layout_i == -1)
8390 : {
8391 16374 : single_layout_i = partition.layout;
8392 16374 : deferred_up_to = partition_i;
8393 : }
8394 17642 : else if (partition.layout == single_layout_i || partition.layout == -1)
8395 : ;
8396 : else
8397 : single_layout_i = 0;
8398 30734 : if (single_layout_i == 0)
8399 : return true;
8400 :
8401 27525 : if (single_layout_i != -1
8402 27525 : && !is_compatible_layout (partition, single_layout_i))
8403 : return true;
8404 : }
8405 :
8406 5945 : if (single_layout_i <= 0)
8407 : return true;
8408 :
8409 6090 : for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
8410 160 : if (!is_compatible_layout (m_partitions[partition_i],
8411 : single_layout_i))
8412 : return true;
8413 :
8414 16806 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8415 : ++partition_i)
8416 : {
8417 10876 : auto &partition = m_partitions[partition_i];
8418 10876 : partition.layout = single_layout_i;
8419 : }
8420 :
8421 : return false;
8422 : }
8423 :
8424 : /* Main entry point for the SLP graph optimization pass. */
8425 :
8426 : void
8427 709983 : vect_optimize_slp_pass::run ()
8428 : {
8429 709983 : build_graph ();
8430 709983 : create_partitions ();
8431 709983 : start_choosing_layouts ();
8432 709983 : if (m_perms.length () > 1)
8433 : {
8434 12525 : if (legitimize ())
8435 : {
8436 6595 : forward_pass ();
8437 6595 : backward_pass ();
8438 : }
8439 12525 : if (dump_enabled_p ())
8440 679 : dump ();
8441 12525 : materialize ();
8442 50678 : while (!m_perms.is_empty ())
8443 25628 : m_perms.pop ().release ();
8444 : }
8445 : else
8446 697458 : remove_redundant_permutations ();
8447 709983 : free_graph (m_slpg);
8448 709983 : build_graph ();
8449 709983 : decide_masked_load_lanes ();
8450 709983 : free_graph (m_slpg);
8451 709983 : }
8452 :
8453 : /* Apply CSE to NODE and its children using BST_MAP. */
8454 :
8455 : static void
8456 5641577 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
8457 : {
8458 5641577 : bool put_p = false;
8459 5641577 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
8460 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8461 : lack scalar stmts and thus CSE doesn't work via bst_map. Ideally
8462 : we'd have sth that works for all internal and external nodes. */
8463 5641577 : && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
8464 : {
8465 4047880 : slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
8466 4047880 : if (leader)
8467 : {
8468 : /* We've visited this node already. */
8469 419381 : if (!*leader || *leader == node)
8470 : return;
8471 :
8472 4262 : if (dump_enabled_p ())
8473 912 : dump_printf_loc (MSG_NOTE, vect_location,
8474 : "re-using SLP tree %p for %p\n",
8475 : (void *)*leader, (void *)node);
8476 4262 : vect_free_slp_tree (node);
8477 4262 : (*leader)->refcnt += 1;
8478 4262 : node = *leader;
8479 4262 : return;
8480 : }
8481 :
8482 : /* Avoid creating a cycle by populating the map only after recursion. */
8483 3628499 : bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
8484 3628499 : node->refcnt += 1;
8485 3628499 : put_p = true;
8486 : /* And recurse. */
8487 : }
8488 :
8489 15608406 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8490 4571352 : if (child)
8491 4116216 : vect_cse_slp_nodes (bst_map, child);
8492 :
8493 : /* Now record the node for CSE in other siblings. */
8494 5222196 : if (put_p)
8495 3628499 : *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
8496 : }
8497 :
8498 : /* Optimize the SLP graph of VINFO. */
8499 :
8500 : void
8501 1094418 : vect_optimize_slp (vec_info *vinfo)
8502 : {
8503 1094418 : if (vinfo->slp_instances.is_empty ())
8504 : return;
8505 709983 : vect_optimize_slp_pass (vinfo).run ();
8506 :
8507 : /* Apply CSE again to nodes after permute optimization. */
8508 709983 : scalar_stmts_to_slp_tree_map_t *bst_map
8509 709983 : = new scalar_stmts_to_slp_tree_map_t ();
8510 :
8511 3655310 : for (auto inst : vinfo->slp_instances)
8512 1525361 : vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
8513 :
8514 709983 : release_scalar_stmts_to_slp_tree_map (bst_map);
8515 : }
8516 :
8517 : /* Gather loads reachable from the individual SLP graph entries. */
8518 :
8519 : void
8520 1094418 : vect_gather_slp_loads (vec_info *vinfo)
8521 : {
8522 1094418 : unsigned i;
8523 1094418 : slp_instance instance;
8524 2619779 : FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
8525 : {
8526 1525361 : hash_set<slp_tree> visited;
8527 1525361 : vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
8528 : SLP_INSTANCE_TREE (instance), visited);
8529 1525361 : }
8530 1094418 : }
8531 :
8532 : /* For NODE update VF based on the number of lanes and the vector types
8533 : used. */
8534 :
8535 : static void
8536 4469595 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
8537 : hash_set<slp_tree> &visited)
8538 : {
8539 4469595 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
8540 1616674 : return;
8541 3227550 : if (visited.add (node))
8542 : return;
8543 :
8544 10861640 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8545 3677509 : vect_update_slp_vf_for_node (child, vf, visited);
8546 :
8547 : /* We do not visit SLP nodes for constants or externals - those neither
8548 : have a vector type set yet (vectorizable_* does this) nor do they
8549 : have max_nunits set. Instead we rely on internal nodes max_nunit
8550 : to cover constant/external operands.
8551 : Note that when we stop using fixed size vectors externs and constants
8552 : shouldn't influence the (minimum) vectorization factor, instead
8553 : vectorizable_* should honor the vectorization factor when trying to
8554 : assign vector types to constants and externals and cause iteration
8555 : to a higher vectorization factor when required. */
8556 2852921 : poly_uint64 node_vf
8557 2852921 : = calculate_unrolling_factor (node->max_nunits, SLP_TREE_LANES (node));
8558 2852921 : vf = force_common_multiple (vf, node_vf);
8559 :
8560 : /* For permute nodes that are fed from externs or constants we have to
8561 : consider their number of lanes as well. Likewise for store-lanes. */
8562 2852921 : if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
8563 717012 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8564 193116 : if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
8565 : {
8566 3599 : poly_uint64 child_vf
8567 3599 : = calculate_unrolling_factor (node->max_nunits,
8568 : SLP_TREE_LANES (child));
8569 3599 : vf = force_common_multiple (vf, child_vf);
8570 : }
8571 : }
8572 :
8573 : /* For each possible SLP instance decide whether to SLP it and calculate overall
8574 : unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at
8575 : least one instance. */
8576 :
8577 : bool
8578 497099 : vect_make_slp_decision (loop_vec_info loop_vinfo)
8579 : {
8580 497099 : unsigned int i;
8581 497099 : poly_uint64 unrolling_factor = 1;
8582 497099 : const vec<slp_instance> &slp_instances
8583 : = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
8584 497099 : slp_instance instance;
8585 497099 : int decided_to_slp = 0;
8586 :
8587 497099 : DUMP_VECT_SCOPE ("vect_make_slp_decision");
8588 :
8589 497099 : hash_set<slp_tree> visited;
8590 1289185 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
8591 : {
8592 792086 : slp_tree root = SLP_INSTANCE_TREE (instance);
8593 :
8594 : /* All unroll factors have the form:
8595 :
8596 : GET_MODE_SIZE (vinfo->vector_mode) * X
8597 :
8598 : for some rational X, so they must have a common multiple. */
8599 792086 : vect_update_slp_vf_for_node (root, unrolling_factor, visited);
8600 :
8601 : /* If all instances ended up with vector(1) T roots make sure to
8602 : not vectorize. RVV for example relies on loop vectorization
8603 : when some instances are essentially kept scalar. See PR121048. */
8604 792086 : if (SLP_TREE_VECTYPE (root)
8605 792086 : && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
8606 643463 : decided_to_slp++;
8607 : }
8608 :
8609 497099 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
8610 :
8611 497099 : if (decided_to_slp && dump_enabled_p ())
8612 : {
8613 19489 : dump_printf_loc (MSG_NOTE, vect_location,
8614 : "Decided to SLP %d instances. Unrolling factor ",
8615 : decided_to_slp);
8616 19489 : dump_dec (MSG_NOTE, unrolling_factor);
8617 19489 : dump_printf (MSG_NOTE, "\n");
8618 : }
8619 :
8620 497099 : return (decided_to_slp > 0);
8621 497099 : }
8622 :
8623 : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks. */
8624 :
8625 2251752 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
8626 : : vec_info (vec_info::bb, shared),
8627 2251752 : roots (vNULL)
8628 : {
8629 : /* The region we are operating on. bbs[0] is the entry, excluding
8630 : its PHI nodes. In the future we might want to track an explicit
8631 : entry edge to cover bbs[0] PHI nodes and have a region entry
8632 : insert location. */
8633 2251752 : bbs = _bbs.address ();
8634 2251752 : nbbs = _bbs.length ();
8635 :
8636 18030778 : for (unsigned i = 0; i < nbbs; ++i)
8637 : {
8638 15779026 : if (i != 0)
8639 20522277 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8640 6995003 : gsi_next (&si))
8641 : {
8642 6995003 : gphi *phi = si.phi ();
8643 6995003 : gimple_set_uid (phi, 0);
8644 6995003 : add_stmt (phi);
8645 : }
8646 31558052 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8647 142319577 : !gsi_end_p (gsi); gsi_next (&gsi))
8648 : {
8649 126540551 : gimple *stmt = gsi_stmt (gsi);
8650 126540551 : gimple_set_uid (stmt, 0);
8651 126540551 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
8652 80688711 : continue;
8653 45851840 : add_stmt (stmt);
8654 : }
8655 : }
8656 2251752 : }
8657 :
8658 :
8659 : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
8660 : stmts in the basic block. */
8661 :
8662 2251752 : _bb_vec_info::~_bb_vec_info ()
8663 : {
8664 : /* Reset region marker. */
8665 18030778 : for (unsigned i = 0; i < nbbs; ++i)
8666 : {
8667 15779026 : if (i != 0)
8668 20538481 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8669 7011207 : gsi_next (&si))
8670 : {
8671 7011207 : gphi *phi = si.phi ();
8672 7011207 : gimple_set_uid (phi, -1);
8673 : }
8674 31558052 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8675 142280674 : !gsi_end_p (gsi); gsi_next (&gsi))
8676 : {
8677 126501648 : gimple *stmt = gsi_stmt (gsi);
8678 126501648 : gimple_set_uid (stmt, -1);
8679 : }
8680 : }
8681 :
8682 3631690 : for (unsigned i = 0; i < roots.length (); ++i)
8683 : {
8684 1379938 : roots[i].stmts.release ();
8685 1379938 : roots[i].roots.release ();
8686 1379938 : roots[i].remain.release ();
8687 : }
8688 2251752 : roots.release ();
8689 2251752 : }
8690 :
8691 : /* Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE,
8692 : given then that child nodes have already been processed, and that
8693 : their def types currently match their SLP node's def type. */
8694 :
8695 : static bool
8696 2716329 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
8697 : slp_instance node_instance,
8698 : stmt_vector_for_cost *cost_vec)
8699 : {
8700 : /* Handle purely internal nodes. */
8701 2716329 : if (SLP_TREE_PERMUTE_P (node))
8702 : {
8703 82375 : if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
8704 : return false;
8705 :
8706 : stmt_vec_info slp_stmt_info;
8707 : unsigned int i;
8708 202780 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
8709 : {
8710 122028 : if (slp_stmt_info
8711 116357 : && STMT_VINFO_LIVE_P (slp_stmt_info)
8712 122028 : && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
8713 : node_instance, i,
8714 : false, cost_vec))
8715 : return false;
8716 : }
8717 80752 : SLP_TREE_TYPE (node) = permute_info_type;
8718 80752 : return true;
8719 : }
8720 :
8721 2633954 : return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
8722 : }
8723 :
8724 : static int
8725 1929307 : sort_ints (const void *a_, const void *b_)
8726 : {
8727 1929307 : int a = *(const int *)a_;
8728 1929307 : int b = *(const int *)b_;
8729 1929307 : return a - b;
8730 : }
8731 :
8732 : /* Verify if we can externalize a set of internal defs. */
8733 :
8734 : static bool
8735 411492 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
8736 : {
8737 : /* Constant generation uses get_later_stmt which can only handle
8738 : defs from the same BB or a set of defs that can be ordered
8739 : with a dominance query. */
8740 411492 : basic_block bb = NULL;
8741 411492 : bool all_same = true;
8742 411492 : auto_vec<int> bbs;
8743 822984 : bbs.reserve_exact (stmts.length ());
8744 2215618 : for (stmt_vec_info stmt : stmts)
8745 : {
8746 981142 : if (!stmt)
8747 : return false;
8748 981142 : else if (!bb)
8749 411492 : bb = gimple_bb (stmt->stmt);
8750 569650 : else if (gimple_bb (stmt->stmt) != bb)
8751 187007 : all_same = false;
8752 981142 : bbs.quick_push (gimple_bb (stmt->stmt)->index);
8753 : }
8754 411492 : if (all_same)
8755 : return true;
8756 :
8757 : /* Produce a vector of unique BB indexes for the defs. */
8758 140100 : bbs.qsort (sort_ints);
8759 : unsigned i, j;
8760 338842 : for (i = 1, j = 1; i < bbs.length (); ++i)
8761 198742 : if (bbs[i] != bbs[j-1])
8762 149669 : bbs[j++] = bbs[i];
8763 140100 : gcc_assert (j >= 2);
8764 140100 : bbs.truncate (j);
8765 :
8766 280200 : if (bbs.length () == 2)
8767 136573 : return (dominated_by_p (CDI_DOMINATORS,
8768 136573 : BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
8769 136573 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
8770 266891 : || dominated_by_p (CDI_DOMINATORS,
8771 130318 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
8772 130318 : BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
8773 :
8774 : /* ??? For more than two BBs we can sort the vector and verify the
8775 : result is a total order. But we can't use vec::qsort with a
8776 : compare function using a dominance query since there's no way to
8777 : signal failure and any fallback for an unordered pair would
8778 : fail qsort_chk later.
8779 : For now simply hope that ordering after BB index provides the
8780 : best candidate total order. If required we can implement our
8781 : own mergesort or export an entry without checking. */
8782 428060 : for (unsigned i = 1; i < bbs.length (); ++i)
8783 13072 : if (!dominated_by_p (CDI_DOMINATORS,
8784 13072 : BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
8785 13072 : BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
8786 : return false;
8787 :
8788 : return true;
8789 411492 : }
8790 :
8791 : /* Try to build NODE from scalars, returning true on success.
8792 : NODE_INSTANCE is the SLP instance that contains NODE. */
8793 :
8794 : static bool
8795 636956 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
8796 : slp_instance node_instance)
8797 : {
8798 636956 : stmt_vec_info stmt_info;
8799 636956 : unsigned int i;
8800 :
8801 636956 : if (!is_a <bb_vec_info> (vinfo)
8802 90910 : || node == SLP_INSTANCE_TREE (node_instance)
8803 31087 : || !SLP_TREE_SCALAR_STMTS (node).exists ()
8804 31046 : || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
8805 : /* Force the mask use to be built from scalars instead. */
8806 23053 : || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
8807 659772 : || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
8808 614140 : return false;
8809 :
8810 22816 : if (dump_enabled_p ())
8811 74 : dump_printf_loc (MSG_NOTE, vect_location,
8812 : "Building vector operands of %p from scalars instead\n",
8813 : (void *) node);
8814 :
8815 : /* Don't remove and free the child nodes here, since they could be
8816 : referenced by other structures. The analysis and scheduling phases
8817 : (need to) ignore child nodes of anything that isn't vect_internal_def. */
8818 22816 : unsigned int group_size = SLP_TREE_LANES (node);
8819 22816 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
8820 : /* Invariants get their vector type from the uses. */
8821 22816 : SLP_TREE_VECTYPE (node) = NULL_TREE;
8822 22816 : SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
8823 22816 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8824 80500 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
8825 : {
8826 57684 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
8827 57684 : SLP_TREE_SCALAR_OPS (node)[i] = lhs;
8828 : }
8829 : return true;
8830 : }
8831 :
8832 : /* Return true if all elements of the slice are the same. */
8833 : bool
8834 449684 : vect_scalar_ops_slice::all_same_p () const
8835 : {
8836 500863 : for (unsigned int i = 1; i < length; ++i)
8837 414528 : if (!operand_equal_p (op (0), op (i)))
8838 : return false;
8839 : return true;
8840 : }
8841 :
8842 : hashval_t
8843 417965 : vect_scalar_ops_slice_hash::hash (const value_type &s)
8844 : {
8845 417965 : hashval_t hash = 0;
8846 1600839 : for (unsigned i = 0; i < s.length; ++i)
8847 1182874 : hash = iterative_hash_expr (s.op (i), hash);
8848 417965 : return hash;
8849 : }
8850 :
8851 : bool
8852 228135 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
8853 : const compare_type &s2)
8854 : {
8855 228135 : if (s1.length != s2.length)
8856 : return false;
8857 393860 : for (unsigned i = 0; i < s1.length; ++i)
8858 343584 : if (!operand_equal_p (s1.op (i), s2.op (i)))
8859 : return false;
8860 : return true;
8861 : }
8862 :
8863 : /* Compute the prologue cost for invariant or constant operands represented
8864 : by NODE. */
8865 :
8866 : static void
8867 1092578 : vect_prologue_cost_for_slp (vec_info *vinfo, slp_tree node,
8868 : stmt_vector_for_cost *cost_vec)
8869 : {
8870 : /* There's a special case of an existing vector, that costs nothing. */
8871 1092578 : if (SLP_TREE_SCALAR_OPS (node).length () == 0
8872 1092578 : && !SLP_TREE_VEC_DEFS (node).is_empty ())
8873 2271 : return;
8874 : /* Without looking at the actual initializer a vector of
8875 : constants can be implemented as load from the constant pool.
8876 : When all elements are the same we can use a splat. */
8877 1090307 : tree vectype = SLP_TREE_VECTYPE (node);
8878 1090307 : unsigned group_size = SLP_TREE_LANES (node);
8879 1090307 : unsigned HOST_WIDE_INT const_nunits;
8880 1090307 : unsigned nelt_limit;
8881 1090307 : unsigned nvectors = vect_get_num_copies (vinfo, node);
8882 1090307 : auto ops = &SLP_TREE_SCALAR_OPS (node);
8883 1090307 : auto_vec<unsigned int> starts (nvectors);
8884 1090307 : if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
8885 1090307 : && ! multiple_p (const_nunits, group_size))
8886 : {
8887 65919 : nelt_limit = const_nunits;
8888 65919 : hash_set<vect_scalar_ops_slice_hash> vector_ops;
8889 273535 : for (unsigned int i = 0; i < nvectors; ++i)
8890 207616 : if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
8891 157340 : starts.quick_push (i * nelt_limit);
8892 65919 : }
8893 : else
8894 : {
8895 : /* If either the vector has variable length or the vectors
8896 : are composed of repeated whole groups we only need to
8897 : cost construction once. All vectors will be the same. */
8898 1024388 : nelt_limit = group_size;
8899 1024388 : starts.quick_push (0);
8900 : }
8901 : /* ??? We're just tracking whether vectors in a single node are the same.
8902 : Ideally we'd do something more global. */
8903 1090307 : bool passed = false;
8904 4452649 : for (unsigned int start : starts)
8905 : {
8906 1181728 : vect_cost_for_stmt kind;
8907 1181728 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
8908 : kind = vector_load;
8909 449684 : else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
8910 : kind = scalar_to_vec;
8911 : else
8912 363349 : kind = vec_construct;
8913 : /* The target cost hook has no idea which part of the SLP node
8914 : we are costing so avoid passing it down more than once. Pass
8915 : it to the first vec_construct or scalar_to_vec part since for those
8916 : the x86 backend tries to account for GPR to XMM register moves. */
8917 1181728 : record_stmt_cost (cost_vec, 1, kind, nullptr,
8918 1181728 : (kind != vector_load && !passed) ? node : nullptr,
8919 : vectype, 0, vect_prologue);
8920 1181728 : if (kind != vector_load)
8921 449684 : passed = true;
8922 : }
8923 1090307 : }
8924 :
8925 : /* Analyze statements contained in SLP tree NODE after recursively analyzing
8926 : the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
8927 :
8928 : Return true if the operations are supported. */
8929 :
8930 : static bool
8931 5043518 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
8932 : slp_instance node_instance,
8933 : hash_set<slp_tree> &visited_set,
8934 : vec<slp_tree> &visited_vec,
8935 : stmt_vector_for_cost *cost_vec)
8936 : {
8937 5043518 : int i, j;
8938 5043518 : slp_tree child;
8939 :
8940 : /* Assume we can code-generate all invariants. */
8941 5043518 : if (!node
8942 4652240 : || SLP_TREE_DEF_TYPE (node) == vect_constant_def
8943 3859765 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
8944 : return true;
8945 :
8946 3374619 : if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
8947 : {
8948 5 : if (dump_enabled_p ())
8949 0 : dump_printf_loc (MSG_NOTE, vect_location,
8950 : "Failed cyclic SLP reference in %p\n", (void *) node);
8951 5 : return false;
8952 : }
8953 3374614 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
8954 :
8955 : /* If we already analyzed the exact same set of scalar stmts we're done.
8956 : We share the generated vector stmts for those. */
8957 3374614 : if (visited_set.add (node))
8958 : return true;
8959 3076384 : visited_vec.safe_push (node);
8960 :
8961 3076384 : bool res = true;
8962 3076384 : unsigned visited_rec_start = visited_vec.length ();
8963 3076384 : unsigned cost_vec_rec_start = cost_vec->length ();
8964 3076384 : bool seen_non_constant_child = false;
8965 6400742 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
8966 : {
8967 3684199 : res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
8968 : visited_set, visited_vec,
8969 : cost_vec);
8970 3684199 : if (!res)
8971 : break;
8972 3324358 : if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
8973 3324358 : seen_non_constant_child = true;
8974 : }
8975 : /* We're having difficulties scheduling nodes with just constant
8976 : operands and no scalar stmts since we then cannot compute a stmt
8977 : insertion place. */
8978 3076384 : if (res
8979 3076384 : && !seen_non_constant_child
8980 3076384 : && SLP_TREE_SCALAR_STMTS (node).is_empty ())
8981 : {
8982 214 : if (dump_enabled_p ())
8983 6 : dump_printf_loc (MSG_NOTE, vect_location,
8984 : "Cannot vectorize all-constant op node %p\n",
8985 : (void *) node);
8986 : res = false;
8987 : }
8988 :
8989 3076170 : if (res)
8990 2716329 : res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
8991 : cost_vec);
8992 : /* If analysis failed we have to pop all recursive visited nodes
8993 : plus ourselves. */
8994 3076384 : if (!res)
8995 : {
8996 3148822 : while (visited_vec.length () >= visited_rec_start)
8997 937455 : visited_set.remove (visited_vec.pop ());
8998 636956 : cost_vec->truncate (cost_vec_rec_start);
8999 : }
9000 :
9001 : /* When the node can be vectorized cost invariant nodes it references.
9002 : This is not done in DFS order to allow the referring node
9003 : vectorizable_* calls to nail down the invariant nodes vector type
9004 : and possibly unshare it if it needs a different vector type than
9005 : other referrers. */
9006 3076384 : if (res)
9007 5408530 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
9008 2969102 : if (child
9009 2649437 : && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
9010 2649437 : || SLP_TREE_DEF_TYPE (child) == vect_external_def)
9011 : /* Perform usual caching, note code-generation still
9012 : code-gens these nodes multiple times but we expect
9013 : to CSE them later. */
9014 4104459 : && !visited_set.add (child))
9015 : {
9016 1132458 : visited_vec.safe_push (child);
9017 : /* ??? After auditing more code paths make a "default"
9018 : and push the vector type from NODE to all children
9019 : if it is not already set. */
9020 : /* Compute the number of vectors to be generated. */
9021 1132458 : tree vector_type = SLP_TREE_VECTYPE (child);
9022 1132458 : if (!vector_type)
9023 : {
9024 : /* Masked loads can have an undefined (default SSA definition)
9025 : else operand. We do not need to cost it. */
9026 39880 : vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
9027 41311 : if (SLP_TREE_TYPE (node) == load_vec_info_type
9028 41311 : && ((ops.length ()
9029 1431 : && TREE_CODE (ops[0]) == SSA_NAME
9030 0 : && SSA_NAME_IS_DEFAULT_DEF (ops[0])
9031 0 : && VAR_P (SSA_NAME_VAR (ops[0])))
9032 1431 : || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
9033 1431 : continue;
9034 :
9035 : /* For shifts with a scalar argument we don't need
9036 : to cost or code-generate anything.
9037 : ??? Represent this more explicitly. */
9038 38449 : gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
9039 : && j == 1);
9040 38449 : continue;
9041 38449 : }
9042 :
9043 : /* And cost them. */
9044 1092578 : vect_prologue_cost_for_slp (vinfo, child, cost_vec);
9045 : }
9046 :
9047 : /* If this node or any of its children can't be vectorized, try pruning
9048 : the tree here rather than felling the whole thing. */
9049 636956 : if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
9050 : {
9051 : /* We'll need to revisit this for invariant costing and number
9052 : of vectorized stmt setting. */
9053 : res = true;
9054 : }
9055 :
9056 : return res;
9057 : }
9058 :
9059 : /* Mark lanes of NODE that are live outside of the basic-block vectorized
9060 : region and that can be vectorized using vectorizable_live_operation
9061 : with STMT_VINFO_LIVE_P. Not handled live operations will cause the
9062 : scalar code computing it to be retained. */
9063 :
9064 : static void
9065 989913 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
9066 : slp_instance instance,
9067 : stmt_vector_for_cost *cost_vec,
9068 : hash_set<stmt_vec_info> &svisited,
9069 : hash_set<slp_tree> &visited)
9070 : {
9071 989913 : if (visited.add (node))
9072 56568 : return;
9073 :
9074 933345 : unsigned i;
9075 933345 : stmt_vec_info stmt_info;
9076 933345 : stmt_vec_info last_stmt = vect_find_last_scalar_stmt_in_slp (node);
9077 3353233 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9078 : {
9079 2419888 : if (!stmt_info || svisited.contains (stmt_info))
9080 90136 : continue;
9081 2384344 : stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
9082 2384344 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
9083 35547 : && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
9084 : /* Only the pattern root stmt computes the original scalar value. */
9085 29615 : continue;
9086 2354729 : if (!PURE_SLP_STMT (orig_stmt_info))
9087 : /* Iff the stmt is not part of the vector coverage because it or
9088 : uses of it are used by SLP graph leafs as extern input there is
9089 : no point in trying to live code-generate from a vector stmt as
9090 : the scalar stmt will survive anyway. */
9091 24977 : continue;
9092 2329752 : bool mark_visited = true;
9093 2329752 : gimple *orig_stmt = orig_stmt_info->stmt;
9094 2329752 : ssa_op_iter op_iter;
9095 2329752 : def_operand_p def_p;
9096 5225818 : FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
9097 : {
9098 : /* We have to verify whether we can insert the lane extract
9099 : before all uses. The following is a conservative approximation.
9100 : We cannot put this into vectorizable_live_operation because
9101 : iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
9102 : doesn't work.
9103 : Note that while the fact that we emit code for loads at the
9104 : first load should make this a non-problem leafs we construct
9105 : from scalars are vectorized after the last scalar def.
9106 : ??? If we'd actually compute the insert location during
9107 : analysis we could use sth less conservative than the last
9108 : scalar stmt in the node for the dominance check. */
9109 : /* ??? What remains is "live" uses in vector CTORs in the same
9110 : SLP graph which is where those uses can end up code-generated
9111 : right after their definition instead of close to their original
9112 : use. But that would restrict us to code-generate lane-extracts
9113 : from the latest stmt in a node. So we compensate for this
9114 : during code-generation, simply not replacing uses for those
9115 : hopefully rare cases. */
9116 566314 : imm_use_iterator use_iter;
9117 566314 : gimple *use_stmt;
9118 566314 : stmt_vec_info use_stmt_info;
9119 :
9120 566314 : bool live_p = false;
9121 566314 : bool can_insert = true;
9122 2188668 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9123 1076496 : if (!is_gimple_debug (use_stmt)
9124 1076496 : && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
9125 825936 : || !PURE_SLP_STMT (use_stmt_info)))
9126 : {
9127 196026 : live_p = true;
9128 196026 : if (!vect_stmt_dominates_stmt_p (last_stmt->stmt, use_stmt))
9129 : {
9130 20456 : if (dump_enabled_p ())
9131 43 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9132 : "Cannot determine insertion place for "
9133 : "lane extract of %T at node %p\n",
9134 : DEF_FROM_PTR (def_p), (void *)node);
9135 : can_insert = false;
9136 : break;
9137 : }
9138 566314 : }
9139 566314 : if (live_p && can_insert)
9140 : {
9141 : /* Only record a live stmt when we can replace all uses. We
9142 : record from which SLP tree we vectorize the uses, so we'll
9143 : cost once and can deal with the case that not all SLP nodes
9144 : may be suitable for code-generation of all live uses.
9145 : ??? But we never split up the work between multiple SLP
9146 : nodes. */
9147 87353 : STMT_VINFO_LIVE_P (stmt_info) = true;
9148 87353 : if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
9149 : instance, i, false, cost_vec))
9150 : {
9151 0 : STMT_VINFO_LIVE_P (stmt_info) = false;
9152 0 : mark_visited = false;
9153 : }
9154 : }
9155 : }
9156 2329752 : if (mark_visited)
9157 2329752 : svisited.add (stmt_info);
9158 : }
9159 :
9160 : slp_tree child;
9161 2729806 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9162 975720 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9163 276778 : vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
9164 : svisited, visited);
9165 : }
9166 :
9167 : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
9168 : are live outside of the basic-block vectorized region and that can be
9169 : vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. */
9170 :
9171 : static void
9172 245217 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
9173 : {
9174 245217 : if (bb_vinfo->slp_instances.is_empty ())
9175 0 : return;
9176 :
9177 245217 : hash_set<slp_tree> visited;
9178 245217 : hash_set<stmt_vec_info> svisited;
9179 1448786 : for (slp_instance instance : bb_vinfo->slp_instances)
9180 : {
9181 713135 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9182 50995 : STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
9183 713135 : vect_location = instance->location ();
9184 713135 : vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
9185 : instance, &instance->cost_vec,
9186 : svisited, visited);
9187 : }
9188 245217 : }
9189 :
9190 : /* Determine whether we can vectorize the reduction epilogue for INSTANCE. */
9191 :
9192 : static bool
9193 89318 : vectorizable_bb_reduc_epilogue (slp_instance instance,
9194 : stmt_vector_for_cost *cost_vec)
9195 : {
9196 89318 : gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
9197 89318 : enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
9198 89318 : if (reduc_code == MINUS_EXPR)
9199 0 : reduc_code = PLUS_EXPR;
9200 89318 : internal_fn reduc_fn;
9201 89318 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
9202 89318 : if (!vectype
9203 89306 : || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
9204 89306 : TREE_TYPE (vectype))
9205 52412 : || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
9206 1041 : && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
9207 1041 : || reduc_fn == IFN_LAST
9208 1041 : || !direct_internal_fn_supported_p (reduc_fn, vectype,
9209 : OPTIMIZE_FOR_BOTH)))
9210 : /* Two-element reductions do not need special-handling for fold-left,
9211 : other cases are not yet implemented. remain_defs also have to
9212 : be included here. */
9213 141728 : || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
9214 3797 : && (!instance->remain_defs.is_empty ()
9215 1860 : || SLP_TREE_LANES (SLP_INSTANCE_TREE (instance)) != 2)))
9216 : {
9217 38895 : if (dump_enabled_p ())
9218 48 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9219 : "not vectorized: basic block reduction epilogue "
9220 : "operation unsupported.\n");
9221 38895 : return false;
9222 : }
9223 :
9224 : /* There's no way to cost a horizontal vector reduction via REDUC_FN so
9225 : cost log2 vector operations plus shuffles and one extraction. */
9226 50423 : unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
9227 50423 : record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
9228 : vectype, 0, vect_body);
9229 50423 : record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
9230 : vectype, 0, vect_body);
9231 50423 : record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
9232 : vectype, 0, vect_body);
9233 :
9234 : /* Since we replace all stmts of a possibly longer scalar reduction
9235 : chain account for the extra scalar stmts for that. */
9236 50423 : if (!instance->remain_defs.is_empty ())
9237 33746 : record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
9238 16873 : instance->root_stmts[0], 0, vect_body);
9239 : return true;
9240 : }
9241 :
9242 : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
9243 : and recurse to children. */
9244 :
9245 : static void
9246 364522 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
9247 : hash_set<slp_tree> &visited)
9248 : {
9249 364522 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
9250 364522 : || visited.add (node))
9251 172154 : return;
9252 :
9253 1012615 : for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
9254 448089 : if (stmt)
9255 479440 : roots.remove (vect_orig_stmt (stmt));
9256 :
9257 772953 : for (auto child : SLP_TREE_CHILDREN (node))
9258 259591 : if (child)
9259 258173 : vect_slp_prune_covered_roots (child, roots, visited);
9260 : }
9261 :
9262 : /* Hand over COST_VEC to the target COSTS grouped by SLP node. */
9263 :
9264 : static void
9265 979993 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
9266 : {
9267 3751600 : for (unsigned start = 0; start < cost_vec.length ();)
9268 : {
9269 2771607 : unsigned end = start + 1;
9270 3422556 : while (end < cost_vec.length ()
9271 5873783 : && cost_vec[start].node == cost_vec[end].node)
9272 650949 : end++;
9273 2771607 : costs->add_slp_cost (cost_vec[start].node,
9274 2771607 : array_slice<stmt_info_for_cost>
9275 2771607 : (cost_vec.begin () + start, end - start));
9276 2771607 : start = end;
9277 : }
9278 979993 : }
9279 :
9280 : /* Analyze statements in SLP instances of VINFO. Return true if the
9281 : operations are supported. */
9282 :
9283 : bool
9284 690566 : vect_slp_analyze_operations (vec_info *vinfo)
9285 : {
9286 690566 : slp_instance instance;
9287 690566 : int i;
9288 :
9289 690566 : DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
9290 :
9291 690566 : hash_set<slp_tree> visited;
9292 1792653 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9293 : {
9294 1359319 : auto_vec<slp_tree> visited_vec;
9295 1359319 : stmt_vector_for_cost cost_vec;
9296 1359319 : cost_vec.create (2);
9297 1359319 : if (is_a <bb_vec_info> (vinfo))
9298 817410 : vect_location = instance->location ();
9299 1359319 : if (!vect_slp_analyze_node_operations (vinfo,
9300 : SLP_INSTANCE_TREE (instance),
9301 : instance, visited, visited_vec,
9302 : &cost_vec)
9303 : /* CTOR instances require vectorized defs for the SLP tree root. */
9304 1105015 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
9305 6129 : && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
9306 : != vect_internal_def
9307 : /* Make sure we vectorized with the expected type. */
9308 6129 : || !useless_type_conversion_p
9309 6129 : (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
9310 : (instance->root_stmts[0]->stmt))),
9311 6129 : TREE_TYPE (SLP_TREE_VECTYPE
9312 : (SLP_INSTANCE_TREE (instance))))))
9313 : /* Check we can vectorize the reduction. */
9314 1105000 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
9315 89318 : && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
9316 : /* Check we can vectorize the gcond. */
9317 2425424 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
9318 65385 : && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
9319 65385 : SLP_INSTANCE_ROOT_STMTS (instance)[0],
9320 : NULL,
9321 : SLP_INSTANCE_TREE (instance),
9322 : &cost_vec)))
9323 : {
9324 355965 : cost_vec.release ();
9325 355965 : slp_tree node = SLP_INSTANCE_TREE (instance);
9326 355965 : stmt_vec_info stmt_info;
9327 355965 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9328 266281 : stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9329 89684 : else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
9330 89684 : && SLP_TREE_SCALAR_STMTS (node)[0])
9331 : stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
9332 : else
9333 0 : stmt_info = SLP_TREE_REPRESENTATIVE (node);
9334 355965 : if (is_a <loop_vec_info> (vinfo))
9335 : {
9336 257232 : if (dump_enabled_p ())
9337 6833 : dump_printf_loc (MSG_NOTE, vect_location,
9338 : "unsupported SLP instance starting from: %G",
9339 : stmt_info->stmt);
9340 257232 : return false;
9341 : }
9342 98733 : if (dump_enabled_p ())
9343 104 : dump_printf_loc (MSG_NOTE, vect_location,
9344 : "removing SLP instance operations starting from: %G",
9345 : stmt_info->stmt);
9346 235076 : while (!visited_vec.is_empty ())
9347 : {
9348 136343 : slp_tree node = visited_vec.pop ();
9349 136343 : SLP_TREE_TYPE (node) = undef_vec_info_type;
9350 136343 : if (node->data)
9351 : {
9352 8474 : delete node->data;
9353 8474 : node->data = nullptr;
9354 : }
9355 136343 : visited.remove (node);
9356 : }
9357 98733 : vect_free_slp_instance (instance);
9358 98733 : vinfo->slp_instances.ordered_remove (i);
9359 : }
9360 : else
9361 : {
9362 1003354 : i++;
9363 1003354 : if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
9364 : {
9365 284677 : add_slp_costs (loop_vinfo->vector_costs, cost_vec);
9366 284677 : cost_vec.release ();
9367 : }
9368 : else
9369 : /* For BB vectorization remember the SLP graph entry
9370 : cost for later. */
9371 718677 : instance->cost_vec = cost_vec;
9372 : }
9373 1359319 : }
9374 :
9375 : /* Now look for SLP instances with a root that are covered by other
9376 : instances and remove them. */
9377 433334 : hash_set<stmt_vec_info> roots;
9378 1799568 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9379 992059 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9380 59159 : roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
9381 433334 : if (!roots.is_empty ())
9382 : {
9383 23524 : visited.empty ();
9384 129873 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9385 106349 : vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
9386 : visited);
9387 129873 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9388 106349 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
9389 59159 : && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
9390 : {
9391 5542 : stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9392 5542 : if (dump_enabled_p ())
9393 283 : dump_printf_loc (MSG_NOTE, vect_location,
9394 : "removing SLP instance operations starting "
9395 : "from: %G", root->stmt);
9396 5542 : vect_free_slp_instance (instance);
9397 5542 : vinfo->slp_instances.ordered_remove (i);
9398 : }
9399 : else
9400 100807 : ++i;
9401 : }
9402 :
9403 866668 : return !vinfo->slp_instances.is_empty ();
9404 1123900 : }
9405 :
9406 : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
9407 : closing the eventual chain. */
9408 :
9409 : static slp_instance
9410 797725 : get_ultimate_leader (slp_instance instance,
9411 : hash_map<slp_instance, slp_instance> &instance_leader)
9412 : {
9413 797725 : auto_vec<slp_instance *, 8> chain;
9414 797725 : slp_instance *tem;
9415 901110 : while (*(tem = instance_leader.get (instance)) != instance)
9416 : {
9417 103385 : chain.safe_push (tem);
9418 103385 : instance = *tem;
9419 : }
9420 901110 : while (!chain.is_empty ())
9421 103385 : *chain.pop () = instance;
9422 797725 : return instance;
9423 797725 : }
9424 :
9425 : namespace {
9426 : /* Subroutine of vect_bb_partition_graph_r. Map KEY to INSTANCE in
9427 : KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
9428 : for KEY. Return true if KEY was already in KEY_TO_INSTANCE.
9429 :
9430 : INSTANCE_LEADER is as for get_ultimate_leader. */
9431 :
9432 : template<typename T>
9433 : bool
9434 3543339 : vect_map_to_instance (slp_instance instance, T key,
9435 : hash_map<T, slp_instance> &key_to_instance,
9436 : hash_map<slp_instance, slp_instance> &instance_leader)
9437 : {
9438 : bool existed_p;
9439 3543339 : slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
9440 3543339 : if (!existed_p)
9441 : ;
9442 236553 : else if (key_instance != instance)
9443 : {
9444 : /* If we're running into a previously marked key make us the
9445 : leader of the current ultimate leader. This keeps the
9446 : leader chain acyclic and works even when the current instance
9447 : connects two previously independent graph parts. */
9448 84590 : slp_instance key_leader
9449 84590 : = get_ultimate_leader (key_instance, instance_leader);
9450 84590 : if (key_leader != instance)
9451 26183 : instance_leader.put (key_leader, instance);
9452 : }
9453 3543339 : key_instance = instance;
9454 3543339 : return existed_p;
9455 : }
9456 : }
9457 :
9458 : /* Worker of vect_bb_partition_graph, recurse on NODE. */
9459 :
9460 : static void
9461 989913 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
9462 : slp_instance instance, slp_tree node,
9463 : hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
9464 : hash_map<slp_tree, slp_instance> &node_to_instance,
9465 : hash_map<slp_instance, slp_instance> &instance_leader)
9466 : {
9467 5512419 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
9468 2553426 : if (stmt_info)
9469 2553426 : vect_map_to_instance (instance, stmt_info, stmt_to_instance,
9470 : instance_leader);
9471 :
9472 989913 : if (vect_map_to_instance (instance, node, node_to_instance,
9473 : instance_leader))
9474 : return;
9475 :
9476 3550547 : for (auto child : SLP_TREE_CHILDREN (node))
9477 975720 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9478 276778 : vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
9479 : node_to_instance, instance_leader);
9480 : }
9481 :
9482 : /* Partition the SLP graph into pieces that can be costed independently. */
9483 :
9484 : static void
9485 245217 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
9486 : {
9487 245217 : DUMP_VECT_SCOPE ("vect_bb_partition_graph");
9488 :
9489 : /* First walk the SLP graph assigning each involved scalar stmt a
9490 : corresponding SLP graph entry and upon visiting a previously
9491 : marked stmt, make the stmts leader the current SLP graph entry. */
9492 245217 : hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
9493 245217 : hash_map<slp_tree, slp_instance> node_to_instance;
9494 245217 : hash_map<slp_instance, slp_instance> instance_leader;
9495 245217 : slp_instance instance;
9496 958352 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9497 : {
9498 713135 : instance_leader.put (instance, instance);
9499 713135 : vect_bb_partition_graph_r (bb_vinfo,
9500 : instance, SLP_INSTANCE_TREE (instance),
9501 : stmt_to_instance, node_to_instance,
9502 : instance_leader);
9503 : }
9504 :
9505 : /* Then collect entries to each independent subgraph. */
9506 1203569 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9507 : {
9508 713135 : slp_instance leader = get_ultimate_leader (instance, instance_leader);
9509 713135 : leader->subgraph_entries.safe_push (instance);
9510 713135 : if (dump_enabled_p ()
9511 713135 : && leader != instance)
9512 71 : dump_printf_loc (MSG_NOTE, vect_location,
9513 : "instance %p is leader of %p\n",
9514 : (void *) leader, (void *) instance);
9515 : }
9516 245217 : }
9517 :
9518 : /* Compute the scalar cost of the SLP node NODE and its children
9519 : and return it. Do not account defs that are marked in LIFE and
9520 : update LIFE according to uses of NODE. */
9521 :
9522 : static void
9523 709179 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
9524 : vec<stmt_vec_info> &worklist,
9525 : stmt_vector_for_cost *cost_vec,
9526 : hash_set<stmt_vec_info> &visited)
9527 : {
9528 3300985 : while (!worklist.is_empty ())
9529 : {
9530 2591806 : stmt_vec_info stmt = worklist.pop ();
9531 2911039 : if (!PURE_SLP_STMT (stmt))
9532 339506 : continue;
9533 :
9534 : /* When the stmt is live but not actually vectorized we have
9535 : to keep the feeding scalar defs. */
9536 2278180 : if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
9537 : {
9538 2187216 : bool live_p = false;
9539 2187216 : ssa_op_iter op_iter;
9540 2187216 : def_operand_p def_p;
9541 4821032 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
9542 : {
9543 446600 : imm_use_iterator use_iter;
9544 446600 : gimple *use_stmt;
9545 1606656 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9546 713456 : if (!is_gimple_debug (use_stmt))
9547 : {
9548 540356 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
9549 540356 : if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
9550 : {
9551 31600 : if (dump_enabled_p ())
9552 : {
9553 82 : dump_printf_loc (MSG_NOTE, vect_location,
9554 : "stmt considered live: %G",
9555 : stmt->stmt);
9556 82 : dump_printf_loc (MSG_NOTE, vect_location,
9557 : "because of use in: %G",
9558 : use_stmt);
9559 : }
9560 : live_p = true;
9561 : }
9562 446600 : }
9563 : }
9564 2187216 : if (live_p)
9565 20273 : continue;
9566 : }
9567 :
9568 : /* The following assert verifies that vect_bb_partition_graph
9569 : partitions the SLP graph in a way that each scalar stmt of
9570 : the coverage of the SLP graph belongs to exactly one subgraph.
9571 : ??? This is currently not guaranteed since the function
9572 : works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
9573 : tripping or scalar stmts costed multiple times, making vectorization
9574 : more profitable than it really is. */
9575 : /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
9576 :
9577 2252300 : if (vect_nop_conversion_p (stmt))
9578 : ;
9579 : /* For single-argument PHIs assume coalescing which means zero
9580 : cost for the scalar and the vector PHIs. This avoids
9581 : artificially favoring the vector path (but may pessimize it
9582 : in some cases). */
9583 2222634 : else if (is_a <gphi *> (stmt->stmt)
9584 2222634 : && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
9585 : ;
9586 : else
9587 : {
9588 2212089 : vect_cost_for_stmt kind;
9589 2212089 : if (STMT_VINFO_DATA_REF (stmt))
9590 : {
9591 1994329 : data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
9592 1994329 : tree base = get_base_address (DR_REF (dr));
9593 : /* When the scalar access is to a non-global not
9594 : address-taken decl that is not BLKmode assume we can
9595 : access it with a single non-load/store instruction. */
9596 1994329 : if (DECL_P (base)
9597 1535245 : && !is_global_var (base)
9598 1458908 : && !TREE_ADDRESSABLE (base)
9599 2546410 : && DECL_MODE (base) != BLKmode)
9600 : kind = scalar_stmt;
9601 1850186 : else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
9602 : kind = scalar_load;
9603 : else
9604 1613102 : kind = scalar_store;
9605 : }
9606 : else
9607 : kind = scalar_stmt;
9608 : /* Cost each scalar stmt only once. */
9609 2212089 : gimple_set_visited (stmt->stmt, true);
9610 2212089 : record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
9611 : }
9612 :
9613 : /* Now walk relevant parts of the SSA use-def graph. */
9614 2252300 : slp_oprnds child_ops (stmt);
9615 4756641 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
9616 : {
9617 2504341 : tree op = child_ops.get_op_for_slp_child (stmt, i);
9618 2504341 : stmt_vec_info def = vinfo->lookup_def (op);
9619 2504341 : if (def && !visited.add (def))
9620 752304 : worklist.safe_push (def);
9621 : }
9622 : }
9623 709179 : }
9624 :
9625 :
9626 : /* Comparator for the loop-index sorted cost vectors. */
9627 :
9628 : static int
9629 19304644 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
9630 : {
9631 19304644 : auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
9632 19304644 : auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
9633 19304644 : if (a->first < b->first)
9634 : return -1;
9635 18205357 : else if (a->first == b->first)
9636 17282624 : return 0;
9637 : return 1;
9638 : }
9639 :
9640 : /* Check if vectorization of the basic block is profitable for the
9641 : subgraph denoted by SLP_INSTANCES. */
9642 :
9643 : static bool
9644 683174 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
9645 : vec<slp_instance> slp_instances,
9646 : loop_p orig_loop)
9647 : {
9648 683174 : slp_instance instance;
9649 683174 : int i;
9650 683174 : unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
9651 683174 : unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
9652 :
9653 683174 : if (dump_enabled_p ())
9654 : {
9655 122 : dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph:\n");
9656 249 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9657 127 : dump_printf_loc (MSG_NOTE, vect_location, " entry instance %p -> "
9658 : "node %p\n", (void *)instance,
9659 127 : (void *)SLP_INSTANCE_TREE (instance));
9660 122 : hash_set<slp_tree> visited;
9661 493 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9662 127 : vect_print_slp_graph (MSG_NOTE, vect_location,
9663 : SLP_INSTANCE_TREE (instance), visited);
9664 122 : }
9665 :
9666 : /* Then DFS walk scalar stmts, performing costing and handling
9667 : still live scalar stmts via the previously computed vector coverage. */
9668 683174 : stmt_vector_for_cost scalar_costs = vNULL;
9669 683174 : stmt_vector_for_cost vector_costs = vNULL;
9670 683174 : hash_set<slp_tree> visited;
9671 683174 : hash_set<stmt_vec_info> svisited;
9672 1392353 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9673 : {
9674 709179 : auto_vec<stmt_vec_info> worklist;
9675 709179 : if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
9676 100524 : record_stmt_cost (&scalar_costs,
9677 50262 : SLP_INSTANCE_ROOT_STMTS (instance).length (),
9678 : scalar_stmt,
9679 50262 : SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
9680 3980873 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
9681 : {
9682 1853336 : stmt = vect_orig_stmt (stmt);
9683 1853336 : if (!svisited.add (stmt))
9684 1839502 : worklist.safe_push (stmt);
9685 : }
9686 709179 : vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
9687 709179 : vector_costs.safe_splice (instance->cost_vec);
9688 709179 : instance->cost_vec.release ();
9689 709179 : }
9690 :
9691 683174 : if (dump_enabled_p ())
9692 122 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
9693 :
9694 : /* When costing non-loop vectorization we need to consider each covered
9695 : loop independently and make sure vectorization is profitable. For
9696 : now we assume a loop may be not entered or executed an arbitrary
9697 : number of iterations (??? static information can provide more
9698 : precise info here) which means we can simply cost each containing
9699 : loops stmts separately. */
9700 :
9701 : /* First produce cost vectors sorted by loop index. */
9702 683174 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9703 683174 : li_scalar_costs (scalar_costs.length ());
9704 683174 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9705 683174 : li_vector_costs (vector_costs.length ());
9706 683174 : stmt_info_for_cost *cost;
9707 2945525 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9708 : {
9709 2262351 : unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9710 2262351 : li_scalar_costs.quick_push (std::make_pair (l, cost));
9711 : }
9712 : /* Use a random used loop as fallback in case the first vector_costs
9713 : entry does not have a stmt_info associated with it. */
9714 683174 : unsigned l = li_scalar_costs[0].first;
9715 2608705 : FOR_EACH_VEC_ELT (vector_costs, i, cost)
9716 : {
9717 : /* We inherit from the previous COST, invariants, externals and
9718 : extracts immediately follow the cost for the related stmt. */
9719 1925531 : if (cost->stmt_info)
9720 1164150 : l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9721 1925531 : li_vector_costs.quick_push (std::make_pair (l, cost));
9722 : }
9723 683174 : li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
9724 683174 : li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
9725 :
9726 : /* Now cost the portions individually. */
9727 : unsigned vi = 0;
9728 : unsigned si = 0;
9729 : bool profitable = true;
9730 1380338 : while (si < li_scalar_costs.length ()
9731 2077544 : && vi < li_vector_costs.length ())
9732 : {
9733 697164 : unsigned sl = li_scalar_costs[si].first;
9734 697164 : unsigned vl = li_vector_costs[vi].first;
9735 697164 : if (sl != vl)
9736 : {
9737 1848 : if (dump_enabled_p ())
9738 2 : dump_printf_loc (MSG_NOTE, vect_location,
9739 : "Scalar %d and vector %d loop part do not "
9740 : "match up, skipping scalar part\n", sl, vl);
9741 : /* Skip the scalar part, assuming zero cost on the vector side. */
9742 2768 : do
9743 : {
9744 2768 : si++;
9745 : }
9746 2768 : while (si < li_scalar_costs.length ()
9747 6136 : && li_scalar_costs[si].first == sl);
9748 1848 : continue;
9749 : }
9750 :
9751 695316 : if (dump_enabled_p ())
9752 135 : dump_printf_loc (MSG_NOTE, vect_location,
9753 : "Scalar cost for part in loop %d\n", sl);
9754 695316 : class vector_costs *scalar_target_cost_data = init_cost (bb_vinfo, true);
9755 2259450 : do
9756 : {
9757 2259450 : add_stmt_cost (scalar_target_cost_data, li_scalar_costs[si].second);
9758 2259450 : si++;
9759 : }
9760 2259450 : while (si < li_scalar_costs.length ()
9761 4531425 : && li_scalar_costs[si].first == sl);
9762 695316 : scalar_target_cost_data->finish_cost (nullptr);
9763 695316 : scalar_cost = scalar_target_cost_data->body_cost ();
9764 :
9765 : /* Complete the target-specific vector cost calculation. */
9766 695316 : if (dump_enabled_p ())
9767 135 : dump_printf_loc (MSG_NOTE, vect_location,
9768 : "Vector cost for part in loop %d\n", vl);
9769 695316 : class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
9770 695316 : auto_vec<stmt_info_for_cost> tem;
9771 1918506 : do
9772 : {
9773 1918506 : tem.safe_push (*li_vector_costs[vi].second);
9774 1918506 : vi++;
9775 : }
9776 1918506 : while (vi < li_vector_costs.length ()
9777 3850668 : && li_vector_costs[vi].first == vl);
9778 695316 : add_slp_costs (vect_target_cost_data, tem);
9779 695316 : vect_target_cost_data->finish_cost (scalar_target_cost_data);
9780 695316 : vec_prologue_cost = vect_target_cost_data->prologue_cost ();
9781 695316 : vec_inside_cost = vect_target_cost_data->body_cost ();
9782 695316 : vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
9783 695316 : delete scalar_target_cost_data;
9784 695316 : delete vect_target_cost_data;
9785 :
9786 695316 : vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
9787 :
9788 695316 : if (dump_enabled_p ())
9789 : {
9790 135 : dump_printf_loc (MSG_NOTE, vect_location,
9791 : "Cost model analysis for part in loop %d:\n", sl);
9792 135 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
9793 : vec_inside_cost + vec_outside_cost);
9794 135 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", scalar_cost);
9795 : }
9796 :
9797 : /* Vectorization is profitable if its cost is more than the cost of scalar
9798 : version. Note that we err on the vector side for equal cost because
9799 : the cost estimate is otherwise quite pessimistic (constant uses are
9800 : free on the scalar side but cost a load on the vector side for
9801 : example). */
9802 695316 : if (vec_outside_cost + vec_inside_cost > scalar_cost)
9803 202921 : profitable = false;
9804 695316 : }
9805 1170087 : if (profitable && vi < li_vector_costs.length ())
9806 : {
9807 1150 : if (dump_enabled_p ())
9808 0 : dump_printf_loc (MSG_NOTE, vect_location,
9809 : "Excess vector cost for part in loop %d:\n",
9810 0 : li_vector_costs[vi].first);
9811 : profitable = false;
9812 : }
9813 :
9814 : /* Unset visited flag. This is delayed when the subgraph is profitable
9815 : and we process the loop for remaining unvectorized if-converted code. */
9816 683174 : if (!orig_loop || !profitable)
9817 2943356 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9818 2260336 : gimple_set_visited (cost->stmt_info->stmt, false);
9819 :
9820 683174 : scalar_costs.release ();
9821 683174 : vector_costs.release ();
9822 :
9823 683174 : return profitable;
9824 683174 : }
9825 :
9826 : /* qsort comparator for lane defs. */
9827 :
9828 : static int
9829 120 : vld_cmp (const void *a_, const void *b_)
9830 : {
9831 120 : auto *a = (const std::pair<unsigned, tree> *)a_;
9832 120 : auto *b = (const std::pair<unsigned, tree> *)b_;
9833 120 : return a->first - b->first;
9834 : }
9835 :
9836 : /* Return true if USE_STMT is a vector lane insert into VEC and set
9837 : *THIS_LANE to the lane number that is set. */
9838 :
9839 : static bool
9840 301 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
9841 : {
9842 301 : gassign *use_ass = dyn_cast <gassign *> (use_stmt);
9843 108 : if (!use_ass
9844 108 : || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
9845 35 : || (vec
9846 35 : ? gimple_assign_rhs1 (use_ass) != vec
9847 24 : : ((vec = gimple_assign_rhs1 (use_ass)), false))
9848 59 : || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
9849 59 : TREE_TYPE (gimple_assign_rhs2 (use_ass)))
9850 59 : || !constant_multiple_p
9851 59 : (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
9852 118 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
9853 : this_lane))
9854 242 : return false;
9855 : return true;
9856 : }
9857 :
9858 : /* Find any vectorizable constructors and add them to the grouped_store
9859 : array. */
9860 :
9861 : static void
9862 2251752 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
9863 : {
9864 18030778 : for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
9865 31558052 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
9866 142319577 : !gsi_end_p (gsi); gsi_next (&gsi))
9867 : {
9868 126540551 : gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
9869 : /* This can be used to start SLP discovery for early breaks for BB early breaks
9870 : when we get that far. */
9871 126540551 : if (!assign)
9872 190348345 : continue;
9873 :
9874 32395296 : tree rhs = gimple_assign_rhs1 (assign);
9875 32395296 : enum tree_code code = gimple_assign_rhs_code (assign);
9876 32395296 : use_operand_p use_p;
9877 32395296 : gimple *use_stmt;
9878 32395296 : if (code == CONSTRUCTOR)
9879 : {
9880 1672959 : if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
9881 67690 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
9882 97663 : CONSTRUCTOR_NELTS (rhs))
9883 44872 : || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
9884 1717811 : || uniform_vector_p (rhs))
9885 1658653 : continue;
9886 :
9887 : unsigned j;
9888 : tree val;
9889 70465 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
9890 56159 : if (TREE_CODE (val) != SSA_NAME
9891 56159 : || !bb_vinfo->lookup_def (val))
9892 : break;
9893 34962 : if (j != CONSTRUCTOR_NELTS (rhs))
9894 3175 : continue;
9895 :
9896 14306 : vec<stmt_vec_info> roots = vNULL;
9897 14306 : roots.safe_push (bb_vinfo->lookup_stmt (assign));
9898 14306 : vec<stmt_vec_info> stmts;
9899 14306 : stmts.create (CONSTRUCTOR_NELTS (rhs));
9900 79820 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
9901 51208 : stmts.quick_push
9902 51208 : (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
9903 14306 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
9904 14306 : stmts, roots));
9905 : }
9906 30722337 : else if (code == BIT_INSERT_EXPR
9907 1041 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
9908 715 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
9909 715 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
9910 712 : && integer_zerop (gimple_assign_rhs3 (assign))
9911 398 : && useless_type_conversion_p
9912 398 : (TREE_TYPE (TREE_TYPE (rhs)),
9913 398 : TREE_TYPE (gimple_assign_rhs2 (assign)))
9914 30723073 : && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
9915 : {
9916 : /* We start to match on insert to lane zero but since the
9917 : inserts need not be ordered we'd have to search both
9918 : the def and the use chains. */
9919 263 : tree vectype = TREE_TYPE (rhs);
9920 263 : unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
9921 263 : auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
9922 263 : auto_sbitmap lanes (nlanes);
9923 263 : bitmap_clear (lanes);
9924 263 : bitmap_set_bit (lanes, 0);
9925 263 : tree def = gimple_assign_lhs (assign);
9926 263 : lane_defs.quick_push
9927 263 : (std::make_pair (0, gimple_assign_rhs2 (assign)));
9928 263 : unsigned lanes_found = 1;
9929 : /* Start with the use chains, the last stmt will be the root. */
9930 263 : stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
9931 263 : vec<stmt_vec_info> roots = vNULL;
9932 263 : roots.safe_push (last);
9933 274 : do
9934 : {
9935 274 : use_operand_p use_p;
9936 274 : gimple *use_stmt;
9937 274 : if (!single_imm_use (def, &use_p, &use_stmt))
9938 : break;
9939 264 : unsigned this_lane;
9940 264 : if (!bb_vinfo->lookup_stmt (use_stmt)
9941 264 : || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
9942 299 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
9943 : break;
9944 35 : if (bitmap_bit_p (lanes, this_lane))
9945 : break;
9946 15 : lanes_found++;
9947 15 : bitmap_set_bit (lanes, this_lane);
9948 15 : gassign *use_ass = as_a <gassign *> (use_stmt);
9949 15 : lane_defs.quick_push (std::make_pair
9950 15 : (this_lane, gimple_assign_rhs2 (use_ass)));
9951 15 : last = bb_vinfo->lookup_stmt (use_ass);
9952 15 : roots.safe_push (last);
9953 15 : def = gimple_assign_lhs (use_ass);
9954 : }
9955 15 : while (lanes_found < nlanes);
9956 263 : if (roots.length () > 1)
9957 7 : std::swap(roots[0], roots[roots.length () - 1]);
9958 263 : if (lanes_found < nlanes)
9959 : {
9960 : /* Now search the def chain. */
9961 259 : def = gimple_assign_rhs1 (assign);
9962 261 : do
9963 : {
9964 261 : if (TREE_CODE (def) != SSA_NAME
9965 261 : || !has_single_use (def))
9966 : break;
9967 57 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
9968 57 : unsigned this_lane;
9969 57 : if (!bb_vinfo->lookup_stmt (def_stmt)
9970 37 : || !vect_slp_is_lane_insert (def_stmt,
9971 : NULL_TREE, &this_lane)
9972 81 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
9973 : break;
9974 24 : if (bitmap_bit_p (lanes, this_lane))
9975 : break;
9976 4 : lanes_found++;
9977 4 : bitmap_set_bit (lanes, this_lane);
9978 8 : lane_defs.quick_push (std::make_pair
9979 4 : (this_lane,
9980 4 : gimple_assign_rhs2 (def_stmt)));
9981 4 : roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
9982 4 : def = gimple_assign_rhs1 (def_stmt);
9983 : }
9984 4 : while (lanes_found < nlanes);
9985 : }
9986 263 : if (lanes_found == nlanes)
9987 : {
9988 : /* Sort lane_defs after the lane index and register the root. */
9989 6 : lane_defs.qsort (vld_cmp);
9990 6 : vec<stmt_vec_info> stmts;
9991 6 : stmts.create (nlanes);
9992 30 : for (unsigned i = 0; i < nlanes; ++i)
9993 24 : stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
9994 6 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
9995 6 : stmts, roots));
9996 : }
9997 : else
9998 257 : roots.release ();
9999 263 : }
10000 30722074 : else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10001 29708573 : && (associative_tree_code (code) || code == MINUS_EXPR)
10002 : /* Ops with constants at the tail can be stripped here. */
10003 6606201 : && TREE_CODE (rhs) == SSA_NAME
10004 6532684 : && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
10005 : /* Should be the chain end. */
10006 33600442 : && (!single_imm_use (gimple_assign_lhs (assign),
10007 : &use_p, &use_stmt)
10008 2290327 : || !is_gimple_assign (use_stmt)
10009 1666950 : || (gimple_assign_rhs_code (use_stmt) != code
10010 1105801 : && ((code != PLUS_EXPR && code != MINUS_EXPR)
10011 605226 : || (gimple_assign_rhs_code (use_stmt)
10012 605226 : != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
10013 : {
10014 : /* We start the match at the end of a possible association
10015 : chain. */
10016 2201562 : auto_vec<chain_op_t> chain;
10017 2201562 : auto_vec<std::pair<tree_code, gimple *> > worklist;
10018 2201562 : auto_vec<gimple *> chain_stmts;
10019 2201562 : gimple *code_stmt = NULL, *alt_code_stmt = NULL;
10020 2201562 : if (code == MINUS_EXPR)
10021 353337 : code = PLUS_EXPR;
10022 2201562 : internal_fn reduc_fn;
10023 2600744 : if (!reduction_fn_for_scalar_code (code, &reduc_fn)
10024 2201562 : || reduc_fn == IFN_LAST)
10025 399182 : continue;
10026 1802380 : vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
10027 : /* ??? */
10028 : code_stmt, alt_code_stmt, &chain_stmts,
10029 : false);
10030 3604760 : if (chain.length () > 1)
10031 : {
10032 : /* Sort the chain according to def_type and operation. */
10033 1802380 : chain.sort (dt_sort_cmp, bb_vinfo);
10034 : /* ??? Now we'd want to strip externals and constants
10035 : but record those to be handled in the epilogue. */
10036 : /* ??? For now do not allow mixing ops or externs/constants. */
10037 1802380 : bool invalid = false;
10038 1802380 : unsigned remain_cnt = 0;
10039 5640637 : for (unsigned i = 0; i < chain.length (); ++i)
10040 : {
10041 4191594 : if (chain[i].code != code)
10042 : {
10043 : invalid = true;
10044 : break;
10045 : }
10046 3838257 : if (chain[i].dt != vect_internal_def
10047 : /* Avoid stmts where the def is not the LHS, like
10048 : ASMs. */
10049 7324753 : || (gimple_get_lhs (bb_vinfo->lookup_def
10050 3486496 : (chain[i].op)->stmt)
10051 3486496 : != chain[i].op))
10052 354713 : remain_cnt++;
10053 : }
10054 1802380 : if (!invalid && chain.length () - remain_cnt > 1)
10055 : {
10056 1365626 : vec<stmt_vec_info> stmts;
10057 1365626 : vec<tree> remain = vNULL;
10058 1365626 : stmts.create (chain.length ());
10059 1365626 : if (remain_cnt > 0)
10060 47179 : remain.create (remain_cnt);
10061 4544215 : for (unsigned i = 0; i < chain.length (); ++i)
10062 : {
10063 3178589 : stmt_vec_info stmt_info;
10064 3178589 : if (chain[i].dt == vect_internal_def
10065 3178589 : && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
10066 3090458 : gimple_get_lhs (stmt_info->stmt) == chain[i].op))
10067 3090374 : stmts.quick_push (stmt_info);
10068 : else
10069 88215 : remain.quick_push (chain[i].op);
10070 : }
10071 1365626 : vec<stmt_vec_info> roots;
10072 1365626 : roots.create (chain_stmts.length ());
10073 3178589 : for (unsigned i = 0; i < chain_stmts.length (); ++i)
10074 1812963 : roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
10075 1365626 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
10076 1365626 : stmts, roots, remain));
10077 : }
10078 : }
10079 2201562 : }
10080 : }
10081 2251752 : }
10082 :
10083 : /* Walk the grouped store chains and replace entries with their
10084 : pattern variant if any. */
10085 :
10086 : static void
10087 663442 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
10088 : {
10089 663442 : stmt_vec_info first_element;
10090 663442 : unsigned i;
10091 :
10092 1569083 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
10093 : {
10094 : /* We also have CTORs in this array. */
10095 905641 : if (!STMT_VINFO_GROUPED_ACCESS (first_element))
10096 0 : continue;
10097 905641 : if (STMT_VINFO_IN_PATTERN_P (first_element))
10098 : {
10099 252 : stmt_vec_info orig = first_element;
10100 252 : first_element = STMT_VINFO_RELATED_STMT (first_element);
10101 252 : DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
10102 252 : DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
10103 252 : DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
10104 252 : DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
10105 252 : vinfo->grouped_stores[i] = first_element;
10106 : }
10107 905641 : stmt_vec_info prev = first_element;
10108 2546414 : while (DR_GROUP_NEXT_ELEMENT (prev))
10109 : {
10110 1640773 : stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
10111 1640773 : if (STMT_VINFO_IN_PATTERN_P (elt))
10112 : {
10113 849 : stmt_vec_info orig = elt;
10114 849 : elt = STMT_VINFO_RELATED_STMT (elt);
10115 849 : DR_GROUP_NEXT_ELEMENT (prev) = elt;
10116 849 : DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
10117 849 : DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
10118 : }
10119 1640773 : DR_GROUP_FIRST_ELEMENT (elt) = first_element;
10120 1640773 : prev = elt;
10121 : }
10122 : }
10123 663442 : }
10124 :
10125 : /* Check if the region described by BB_VINFO can be vectorized, returning
10126 : true if so. When returning false, set FATAL to true if the same failure
10127 : would prevent vectorization at other vector sizes, false if it is still
10128 : worth trying other sizes. N_STMTS is the number of statements in the
10129 : region. */
10130 :
10131 : static bool
10132 2251752 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
10133 : vec<int> *dataref_groups)
10134 : {
10135 2251752 : DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
10136 :
10137 2251752 : slp_instance instance;
10138 2251752 : int i;
10139 :
10140 : /* The first group of checks is independent of the vector size. */
10141 2251752 : fatal = true;
10142 :
10143 : /* Analyze the data references. */
10144 :
10145 2251752 : if (!vect_analyze_data_refs (bb_vinfo, NULL))
10146 : {
10147 0 : if (dump_enabled_p ())
10148 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10149 : "not vectorized: unhandled data-ref in basic "
10150 : "block.\n");
10151 0 : return false;
10152 : }
10153 :
10154 2251752 : if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
10155 : {
10156 0 : if (dump_enabled_p ())
10157 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10158 : "not vectorized: unhandled data access in "
10159 : "basic block.\n");
10160 0 : return false;
10161 : }
10162 :
10163 2251752 : vect_slp_check_for_roots (bb_vinfo);
10164 :
10165 : /* If there are no grouped stores and no constructors in the region
10166 : there is no need to continue with pattern recog as vect_analyze_slp
10167 : will fail anyway. */
10168 2251752 : if (bb_vinfo->grouped_stores.is_empty ()
10169 1903664 : && bb_vinfo->roots.is_empty ())
10170 : {
10171 1588310 : if (dump_enabled_p ())
10172 1012 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10173 : "not vectorized: no grouped stores in "
10174 : "basic block.\n");
10175 1588310 : return false;
10176 : }
10177 :
10178 : /* While the rest of the analysis below depends on it in some way. */
10179 663442 : fatal = false;
10180 :
10181 663442 : vect_pattern_recog (bb_vinfo);
10182 :
10183 : /* Update store groups from pattern processing. */
10184 663442 : vect_fixup_store_groups_with_patterns (bb_vinfo);
10185 :
10186 : /* Check the SLP opportunities in the basic block, analyze and build SLP
10187 : trees. */
10188 663442 : if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
10189 : {
10190 0 : if (dump_enabled_p ())
10191 : {
10192 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10193 : "Failed to SLP the basic block.\n");
10194 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10195 : "not vectorized: failed to find SLP opportunities "
10196 : "in basic block.\n");
10197 : }
10198 0 : return false;
10199 : }
10200 :
10201 : /* Optimize permutations. */
10202 663442 : vect_optimize_slp (bb_vinfo);
10203 :
10204 : /* Gather the loads reachable from the SLP graph entries. */
10205 663442 : vect_gather_slp_loads (bb_vinfo);
10206 :
10207 663442 : vect_record_base_alignments (bb_vinfo);
10208 :
10209 : /* Analyze and verify the alignment of data references and the
10210 : dependence in the SLP instances. */
10211 1489385 : for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
10212 : {
10213 825943 : vect_location = instance->location ();
10214 825943 : if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
10215 825943 : || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
10216 : {
10217 8533 : slp_tree node = SLP_INSTANCE_TREE (instance);
10218 8533 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10219 8533 : if (dump_enabled_p ())
10220 4 : dump_printf_loc (MSG_NOTE, vect_location,
10221 : "removing SLP instance operations starting from: %G",
10222 : stmt_info->stmt);
10223 8533 : vect_free_slp_instance (instance);
10224 8533 : BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
10225 8533 : continue;
10226 8533 : }
10227 :
10228 : /* Mark all the statements that we want to vectorize as relevant. */
10229 817410 : vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
10230 :
10231 817410 : i++;
10232 : }
10233 2283668 : if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
10234 : return false;
10235 :
10236 277133 : if (!vect_slp_analyze_operations (bb_vinfo))
10237 : {
10238 31916 : if (dump_enabled_p ())
10239 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10240 : "not vectorized: bad operation in basic block.\n");
10241 31916 : return false;
10242 : }
10243 :
10244 : /* Mark all the statements that we vectorize. */
10245 245217 : vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
10246 :
10247 : /* Compute vectorizable live stmts. */
10248 245217 : vect_bb_slp_mark_live_stmts (bb_vinfo);
10249 :
10250 245217 : vect_bb_partition_graph (bb_vinfo);
10251 :
10252 245217 : return true;
10253 : }
10254 :
10255 : /* Subroutine of vect_slp_bb. Try to vectorize the statements for all
10256 : basic blocks in BBS, returning true on success.
10257 : The region has N_STMTS statements and has the datarefs given by DATAREFS. */
10258 :
10259 : static bool
10260 1898616 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
10261 : vec<int> *dataref_groups, unsigned int n_stmts,
10262 : loop_p orig_loop)
10263 : {
10264 1898616 : bb_vec_info bb_vinfo;
10265 1898616 : auto_vector_modes vector_modes;
10266 :
10267 : /* Autodetect first vector size we try. */
10268 1898616 : machine_mode next_vector_mode = VOIDmode;
10269 1898616 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
10270 1898616 : unsigned int mode_i = 0;
10271 :
10272 1898616 : vec_info_shared shared;
10273 :
10274 1898616 : machine_mode autodetected_vector_mode = VOIDmode;
10275 2604888 : while (1)
10276 : {
10277 2251752 : bool vectorized = false;
10278 2251752 : bool fatal = false;
10279 2251752 : bb_vinfo = new _bb_vec_info (bbs, &shared);
10280 :
10281 2251752 : bool first_time_p = shared.datarefs.is_empty ();
10282 2251752 : BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
10283 2251752 : if (first_time_p)
10284 1923054 : bb_vinfo->shared->save_datarefs ();
10285 : else
10286 328698 : bb_vinfo->shared->check_datarefs ();
10287 2251752 : bb_vinfo->vector_mode = next_vector_mode;
10288 :
10289 2251752 : if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
10290 : {
10291 245217 : if (dump_enabled_p ())
10292 : {
10293 1576 : dump_printf_loc (MSG_NOTE, vect_location,
10294 : "***** Analysis succeeded with vector mode"
10295 788 : " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
10296 788 : dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
10297 : }
10298 :
10299 245217 : bb_vinfo->shared->check_datarefs ();
10300 :
10301 245217 : bool force_clear = false;
10302 245217 : auto_vec<slp_instance> profitable_subgraphs;
10303 1448786 : for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
10304 : {
10305 713135 : if (instance->subgraph_entries.is_empty ())
10306 249764 : continue;
10307 :
10308 686952 : dump_user_location_t saved_vect_location = vect_location;
10309 686952 : vect_location = instance->location ();
10310 686952 : if (!unlimited_cost_model (NULL)
10311 683183 : && !param_vect_allow_possibly_not_worthwhile_vectorizations
10312 1370126 : && !vect_bb_vectorization_profitable_p
10313 683174 : (bb_vinfo, instance->subgraph_entries, orig_loop))
10314 : {
10315 197398 : if (dump_enabled_p ())
10316 49 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10317 : "not vectorized: vectorization is not "
10318 : "profitable.\n");
10319 197398 : vect_location = saved_vect_location;
10320 197398 : continue;
10321 : }
10322 :
10323 489554 : vect_location = saved_vect_location;
10324 489554 : if (!dbg_cnt (vect_slp))
10325 : {
10326 0 : force_clear = true;
10327 0 : continue;
10328 : }
10329 :
10330 489554 : profitable_subgraphs.safe_push (instance);
10331 : }
10332 :
10333 : /* When we're vectorizing an if-converted loop body make sure
10334 : we vectorized all if-converted code. */
10335 407125 : if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
10336 : {
10337 153 : gcc_assert (bb_vinfo->nbbs == 1);
10338 306 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
10339 6049 : !gsi_end_p (gsi); gsi_next (&gsi))
10340 : {
10341 : /* The costing above left us with DCEable vectorized scalar
10342 : stmts having the visited flag set on profitable
10343 : subgraphs. Do the delayed clearing of the flag here. */
10344 5896 : if (gimple_visited_p (gsi_stmt (gsi)))
10345 : {
10346 1876 : gimple_set_visited (gsi_stmt (gsi), false);
10347 1876 : continue;
10348 : }
10349 4020 : if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
10350 813 : continue;
10351 :
10352 8821 : if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
10353 3647 : if (gimple_assign_rhs_code (ass) == COND_EXPR)
10354 : {
10355 157 : if (!profitable_subgraphs.is_empty ()
10356 68 : && dump_enabled_p ())
10357 0 : dump_printf_loc (MSG_NOTE, vect_location,
10358 : "not profitable because of "
10359 : "unprofitable if-converted scalar "
10360 : "code\n");
10361 89 : profitable_subgraphs.truncate (0);
10362 : }
10363 : }
10364 : }
10365 :
10366 : /* Finally schedule the profitable subgraphs. */
10367 1058500 : for (slp_instance instance : profitable_subgraphs)
10368 : {
10369 489467 : if (!vectorized && dump_enabled_p ())
10370 749 : dump_printf_loc (MSG_NOTE, vect_location,
10371 : "Basic block will be vectorized "
10372 : "using SLP\n");
10373 489467 : vectorized = true;
10374 :
10375 : /* Dump before scheduling as store vectorization will remove
10376 : the original stores and mess with the instance tree
10377 : so querying its location will eventually ICE. */
10378 489467 : if (flag_checking)
10379 1969756 : for (slp_instance sub : instance->subgraph_entries)
10380 501355 : gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
10381 489467 : unsigned HOST_WIDE_INT bytes;
10382 489467 : if (dump_enabled_p ())
10383 3561 : for (slp_instance sub : instance->subgraph_entries)
10384 : {
10385 942 : tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
10386 1884 : if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
10387 942 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10388 942 : sub->location (),
10389 : "basic block part vectorized using %wu "
10390 : "byte vectors\n", bytes);
10391 : else
10392 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10393 : sub->location (),
10394 : "basic block part vectorized using "
10395 : "variable length vectors\n");
10396 : }
10397 :
10398 489467 : dump_user_location_t saved_vect_location = vect_location;
10399 489467 : vect_location = instance->location ();
10400 :
10401 489467 : vect_schedule_slp (bb_vinfo, instance->subgraph_entries);
10402 :
10403 489467 : vect_location = saved_vect_location;
10404 : }
10405 :
10406 :
10407 : /* Generate the invariant statements. */
10408 245217 : if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
10409 : {
10410 27 : if (dump_enabled_p ())
10411 0 : dump_printf_loc (MSG_NOTE, vect_location,
10412 : "------>generating invariant statements\n");
10413 :
10414 27 : bb_vinfo->insert_seq_on_entry (NULL,
10415 : bb_vinfo->inv_pattern_def_seq);
10416 : }
10417 245217 : }
10418 : else
10419 : {
10420 2006535 : if (dump_enabled_p ())
10421 1303 : dump_printf_loc (MSG_NOTE, vect_location,
10422 : "***** Analysis failed with vector mode %s\n",
10423 1303 : GET_MODE_NAME (bb_vinfo->vector_mode));
10424 : }
10425 :
10426 2251752 : if (mode_i == 0)
10427 1898616 : autodetected_vector_mode = bb_vinfo->vector_mode;
10428 :
10429 2251752 : if (!fatal)
10430 3269912 : while (mode_i < vector_modes.length ()
10431 1887806 : && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
10432 : {
10433 354718 : if (dump_enabled_p ())
10434 1708 : dump_printf_loc (MSG_NOTE, vect_location,
10435 : "***** The result for vector mode %s would"
10436 : " be the same\n",
10437 854 : GET_MODE_NAME (vector_modes[mode_i]));
10438 354718 : mode_i += 1;
10439 : }
10440 :
10441 2251752 : delete bb_vinfo;
10442 :
10443 2251752 : if (mode_i < vector_modes.length ()
10444 2057690 : && VECTOR_MODE_P (autodetected_vector_mode)
10445 2055872 : && (related_vector_mode (vector_modes[mode_i],
10446 : GET_MODE_INNER (autodetected_vector_mode))
10447 1027936 : == autodetected_vector_mode)
10448 4309442 : && (related_vector_mode (autodetected_vector_mode,
10449 512599 : GET_MODE_INNER (vector_modes[mode_i]))
10450 1025198 : == vector_modes[mode_i]))
10451 : {
10452 512599 : if (dump_enabled_p ())
10453 193 : dump_printf_loc (MSG_NOTE, vect_location,
10454 : "***** Skipping vector mode %s, which would"
10455 : " repeat the analysis for %s\n",
10456 193 : GET_MODE_NAME (vector_modes[mode_i]),
10457 193 : GET_MODE_NAME (autodetected_vector_mode));
10458 512599 : mode_i += 1;
10459 : }
10460 :
10461 2251752 : if (vectorized
10462 2089912 : || mode_i == vector_modes.length ()
10463 1895898 : || autodetected_vector_mode == VOIDmode
10464 : /* If vect_slp_analyze_bb_1 signaled that analysis for all
10465 : vector sizes will fail do not bother iterating. */
10466 3117896 : || fatal)
10467 3797232 : return vectorized;
10468 :
10469 : /* Try the next biggest vector size. */
10470 353136 : next_vector_mode = vector_modes[mode_i++];
10471 353136 : if (dump_enabled_p ())
10472 227 : dump_printf_loc (MSG_NOTE, vect_location,
10473 : "***** Re-trying analysis with vector mode %s\n",
10474 227 : GET_MODE_NAME (next_vector_mode));
10475 353136 : }
10476 1898616 : }
10477 :
10478 :
10479 : /* Main entry for the BB vectorizer. Analyze and transform BBS, returns
10480 : true if anything in the basic-block was vectorized. */
10481 :
10482 : static bool
10483 1898616 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
10484 : {
10485 1898616 : vec<data_reference_p> datarefs = vNULL;
10486 1898616 : auto_vec<int> dataref_groups;
10487 1898616 : int insns = 0;
10488 1898616 : int current_group = 0;
10489 :
10490 12610412 : for (unsigned i = 0; i < bbs.length (); i++)
10491 : {
10492 10711796 : basic_block bb = bbs[i];
10493 92068999 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
10494 81357203 : gsi_next (&gsi))
10495 : {
10496 81357203 : gimple *stmt = gsi_stmt (gsi);
10497 81357203 : if (is_gimple_debug (stmt))
10498 51028113 : continue;
10499 :
10500 30329090 : insns++;
10501 :
10502 30329090 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
10503 27219941 : vect_location = stmt;
10504 :
10505 30329090 : if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
10506 : &dataref_groups, current_group))
10507 5175397 : ++current_group;
10508 : }
10509 : /* New BBs always start a new DR group. */
10510 10711796 : ++current_group;
10511 : }
10512 :
10513 1898616 : return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
10514 1898616 : }
10515 :
10516 : /* Special entry for the BB vectorizer. Analyze and transform a single
10517 : if-converted BB with ORIG_LOOPs body being the not if-converted
10518 : representation. Returns true if anything in the basic-block was
10519 : vectorized. */
10520 :
10521 : bool
10522 19445 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
10523 : {
10524 19445 : auto_vec<basic_block> bbs;
10525 19445 : bbs.safe_push (bb);
10526 19445 : return vect_slp_bbs (bbs, orig_loop);
10527 19445 : }
10528 :
10529 : /* Main entry for the BB vectorizer. Analyze and transform BB, returns
10530 : true if anything in the basic-block was vectorized. */
10531 :
10532 : bool
10533 914389 : vect_slp_function (function *fun)
10534 : {
10535 914389 : bool r = false;
10536 914389 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
10537 914389 : auto_bitmap exit_bbs;
10538 914389 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
10539 914389 : edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
10540 914389 : unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
10541 914389 : true, rpo, NULL);
10542 :
10543 : /* For the moment split the function into pieces to avoid making
10544 : the iteration on the vector mode moot. Split at points we know
10545 : to not handle well which is CFG merges (SLP discovery doesn't
10546 : handle non-loop-header PHIs) and loop exits. Since pattern
10547 : recog requires reverse iteration to visit uses before defs
10548 : simply chop RPO into pieces. */
10549 914389 : auto_vec<basic_block> bbs;
10550 11637665 : for (unsigned i = 0; i < n; i++)
10551 : {
10552 10723276 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
10553 10723276 : bool split = false;
10554 :
10555 : /* Split when a BB is not dominated by the first block. */
10556 20224021 : if (!bbs.is_empty ()
10557 9500745 : && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
10558 : {
10559 676425 : if (dump_enabled_p ())
10560 146 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10561 : "splitting region at dominance boundary bb%d\n",
10562 : bb->index);
10563 : split = true;
10564 : }
10565 : /* Split when the loop determined by the first block
10566 : is exited. This is because we eventually insert
10567 : invariants at region begin. */
10568 18871171 : else if (!bbs.is_empty ()
10569 8824320 : && bbs[0]->loop_father != bb->loop_father
10570 2299697 : && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
10571 : {
10572 3844 : if (dump_enabled_p ())
10573 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10574 : "splitting region at loop %d exit at bb%d\n",
10575 3 : bbs[0]->loop_father->num, bb->index);
10576 : split = true;
10577 : }
10578 10043007 : else if (!bbs.is_empty ()
10579 8820476 : && bb->loop_father->header == bb
10580 472683 : && bb->loop_father->dont_vectorize)
10581 : {
10582 7296 : if (dump_enabled_p ())
10583 72 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10584 : "splitting region at dont-vectorize loop %d "
10585 : "entry at bb%d\n",
10586 : bb->loop_father->num, bb->index);
10587 : split = true;
10588 : }
10589 :
10590 11410841 : if (split && !bbs.is_empty ())
10591 : {
10592 687565 : r |= vect_slp_bbs (bbs, NULL);
10593 687565 : bbs.truncate (0);
10594 : }
10595 :
10596 10723276 : if (bbs.is_empty ())
10597 : {
10598 : /* We need to be able to insert at the head of the region which
10599 : we cannot for region starting with a returns-twice call. */
10600 1910096 : if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
10601 403108 : if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
10602 : {
10603 306 : if (dump_enabled_p ())
10604 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10605 : "skipping bb%d as start of region as it "
10606 : "starts with returns-twice call\n",
10607 : bb->index);
10608 30925 : continue;
10609 : }
10610 : /* If the loop this BB belongs to is marked as not to be vectorized
10611 : honor that also for BB vectorization. */
10612 1909790 : if (bb->loop_father->dont_vectorize)
10613 30619 : continue;
10614 : }
10615 :
10616 10692351 : bbs.safe_push (bb);
10617 :
10618 : /* When we have a stmt ending this block and defining a
10619 : value we have to insert on edges when inserting after it for
10620 : a vector containing its definition. Avoid this for now. */
10621 21384702 : if (gimple *last = *gsi_last_bb (bb))
10622 8684558 : if (gimple_get_lhs (last)
10623 8684558 : && is_ctrl_altering_stmt (last))
10624 : {
10625 277224 : if (dump_enabled_p ())
10626 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10627 : "splitting region at control altering "
10628 : "definition %G", last);
10629 277224 : r |= vect_slp_bbs (bbs, NULL);
10630 277224 : bbs.truncate (0);
10631 : }
10632 : }
10633 :
10634 914389 : if (!bbs.is_empty ())
10635 914382 : r |= vect_slp_bbs (bbs, NULL);
10636 :
10637 914389 : free (rpo);
10638 :
10639 914389 : return r;
10640 914389 : }
10641 :
10642 : /* Build a variable-length vector in which the elements in ELTS are repeated
10643 : to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
10644 : RESULTS and add any new instructions to SEQ.
10645 :
10646 : The approach we use is:
10647 :
10648 : (1) Find a vector mode VM with integer elements of mode IM.
10649 :
10650 : (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10651 : ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
10652 : from small vectors to IM.
10653 :
10654 : (3) Duplicate each ELTS'[I] into a vector of mode VM.
10655 :
10656 : (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
10657 : correct byte contents.
10658 :
10659 : (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
10660 :
10661 : We try to find the largest IM for which this sequence works, in order
10662 : to cut down on the number of interleaves. */
10663 :
10664 : void
10665 0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
10666 : const vec<tree> &elts, unsigned int nresults,
10667 : vec<tree> &results)
10668 : {
10669 0 : unsigned int nelts = elts.length ();
10670 0 : tree element_type = TREE_TYPE (vector_type);
10671 :
10672 : /* (1) Find a vector mode VM with integer elements of mode IM. */
10673 0 : unsigned int nvectors = 1;
10674 0 : tree new_vector_type;
10675 0 : tree permutes[2];
10676 0 : if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
10677 : &nvectors, &new_vector_type,
10678 : permutes))
10679 0 : gcc_unreachable ();
10680 :
10681 : /* Get a vector type that holds ELTS[0:NELTS/NELTS']. */
10682 0 : unsigned int partial_nelts = nelts / nvectors;
10683 0 : tree partial_vector_type = build_vector_type (element_type, partial_nelts);
10684 :
10685 0 : tree_vector_builder partial_elts;
10686 0 : auto_vec<tree, 32> pieces (nvectors * 2);
10687 0 : pieces.quick_grow_cleared (nvectors * 2);
10688 0 : for (unsigned int i = 0; i < nvectors; ++i)
10689 : {
10690 : /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10691 : ELTS' has mode IM. */
10692 0 : partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
10693 0 : for (unsigned int j = 0; j < partial_nelts; ++j)
10694 0 : partial_elts.quick_push (elts[i * partial_nelts + j]);
10695 0 : tree t = gimple_build_vector (seq, &partial_elts);
10696 0 : t = gimple_build (seq, VIEW_CONVERT_EXPR,
10697 0 : TREE_TYPE (new_vector_type), t);
10698 :
10699 : /* (3) Duplicate each ELTS'[I] into a vector of mode VM. */
10700 0 : pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
10701 : }
10702 :
10703 : /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
10704 : correct byte contents.
10705 :
10706 : Conceptually, we need to repeat the following operation log2(nvectors)
10707 : times, where hi_start = nvectors / 2:
10708 :
10709 : out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
10710 : out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
10711 :
10712 : However, if each input repeats every N elements and the VF is
10713 : a multiple of N * 2, the HI result is the same as the LO result.
10714 : This will be true for the first N1 iterations of the outer loop,
10715 : followed by N2 iterations for which both the LO and HI results
10716 : are needed. I.e.:
10717 :
10718 : N1 + N2 = log2(nvectors)
10719 :
10720 : Each "N1 iteration" doubles the number of redundant vectors and the
10721 : effect of the process as a whole is to have a sequence of nvectors/2**N1
10722 : vectors that repeats 2**N1 times. Rather than generate these redundant
10723 : vectors, we halve the number of vectors for each N1 iteration. */
10724 : unsigned int in_start = 0;
10725 : unsigned int out_start = nvectors;
10726 : unsigned int new_nvectors = nvectors;
10727 0 : for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
10728 : {
10729 0 : unsigned int hi_start = new_nvectors / 2;
10730 0 : unsigned int out_i = 0;
10731 0 : for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
10732 : {
10733 0 : if ((in_i & 1) != 0
10734 0 : && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
10735 : 2 * in_repeat))
10736 0 : continue;
10737 :
10738 0 : tree output = make_ssa_name (new_vector_type);
10739 0 : tree input1 = pieces[in_start + (in_i / 2)];
10740 0 : tree input2 = pieces[in_start + (in_i / 2) + hi_start];
10741 0 : gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
10742 : input1, input2,
10743 : permutes[in_i & 1]);
10744 0 : gimple_seq_add_stmt (seq, stmt);
10745 0 : pieces[out_start + out_i] = output;
10746 0 : out_i += 1;
10747 : }
10748 0 : std::swap (in_start, out_start);
10749 0 : new_nvectors = out_i;
10750 : }
10751 :
10752 : /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type. */
10753 0 : results.reserve (nresults);
10754 0 : for (unsigned int i = 0; i < nresults; ++i)
10755 0 : if (i < new_nvectors)
10756 0 : results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
10757 0 : pieces[in_start + i]));
10758 : else
10759 0 : results.quick_push (results[i - new_nvectors]);
10760 0 : }
10761 :
10762 :
10763 : /* For constant and loop invariant defs in OP_NODE this function creates
10764 : vector defs that will be used in the vectorized stmts and stores them
10765 : to SLP_TREE_VEC_DEFS of OP_NODE. */
10766 :
10767 : static void
10768 498514 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
10769 : {
10770 498514 : unsigned HOST_WIDE_INT nunits;
10771 498514 : tree vec_cst;
10772 498514 : unsigned j, number_of_places_left_in_vector;
10773 498514 : tree vector_type;
10774 498514 : tree vop;
10775 498514 : int group_size = op_node->ops.length ();
10776 498514 : unsigned int vec_num, i;
10777 498514 : unsigned number_of_copies = 1;
10778 498514 : bool constant_p;
10779 498514 : gimple_seq ctor_seq = NULL;
10780 498514 : auto_vec<tree, 16> permute_results;
10781 :
10782 : /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
10783 498514 : vector_type = SLP_TREE_VECTYPE (op_node);
10784 :
10785 498514 : unsigned int number_of_vectors = vect_get_num_copies (vinfo, op_node);
10786 498514 : SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
10787 498514 : auto_vec<tree> voprnds (number_of_vectors);
10788 :
10789 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
10790 : created vectors. It is greater than 1 if unrolling is performed.
10791 :
10792 : For example, we have two scalar operands, s1 and s2 (e.g., group of
10793 : strided accesses of size two), while NUNITS is four (i.e., four scalars
10794 : of this type can be packed in a vector). The output vector will contain
10795 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
10796 : will be 2).
10797 :
10798 : If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
10799 : containing the operands.
10800 :
10801 : For example, NUNITS is four as before, and the group size is 8
10802 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
10803 : {s5, s6, s7, s8}. */
10804 :
10805 : /* When using duplicate_and_interleave, we just need one element for
10806 : each scalar statement. */
10807 498514 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
10808 : nunits = group_size;
10809 :
10810 498514 : number_of_copies = nunits * number_of_vectors / group_size;
10811 :
10812 498514 : number_of_places_left_in_vector = nunits;
10813 498514 : constant_p = true;
10814 498514 : tree uniform_elt = NULL_TREE;
10815 498514 : tree_vector_builder elts (vector_type, nunits, 1);
10816 498514 : elts.quick_grow (nunits);
10817 498514 : stmt_vec_info insert_after = NULL;
10818 1481362 : for (j = 0; j < number_of_copies; j++)
10819 : {
10820 982848 : tree op;
10821 3772406 : for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
10822 : {
10823 : /* Create 'vect_ = {op0,op1,...,opn}'. */
10824 1806710 : tree orig_op = op;
10825 1806710 : if (number_of_places_left_in_vector == nunits)
10826 : uniform_elt = op;
10827 1177370 : else if (uniform_elt && operand_equal_p (uniform_elt, op))
10828 746279 : op = elts[number_of_places_left_in_vector];
10829 : else
10830 : uniform_elt = NULL_TREE;
10831 1806710 : number_of_places_left_in_vector--;
10832 1806710 : if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
10833 : {
10834 280771 : if (CONSTANT_CLASS_P (op))
10835 : {
10836 102599 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10837 : {
10838 : /* Can't use VIEW_CONVERT_EXPR for booleans because
10839 : of possibly different sizes of scalar value and
10840 : vector element. */
10841 66 : if (integer_zerop (op))
10842 66 : op = build_int_cst (TREE_TYPE (vector_type), 0);
10843 0 : else if (integer_onep (op))
10844 0 : op = build_all_ones_cst (TREE_TYPE (vector_type));
10845 : else
10846 0 : gcc_unreachable ();
10847 : }
10848 : else
10849 102533 : op = fold_unary (VIEW_CONVERT_EXPR,
10850 : TREE_TYPE (vector_type), op);
10851 102599 : gcc_assert (op && CONSTANT_CLASS_P (op));
10852 : }
10853 : else
10854 : {
10855 178172 : tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
10856 178172 : gimple *init_stmt;
10857 178172 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10858 : {
10859 427 : tree true_val
10860 427 : = build_all_ones_cst (TREE_TYPE (vector_type));
10861 427 : tree false_val
10862 427 : = build_zero_cst (TREE_TYPE (vector_type));
10863 427 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
10864 427 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
10865 : op, true_val,
10866 : false_val);
10867 : }
10868 : else
10869 : {
10870 177745 : op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
10871 : op);
10872 177745 : init_stmt
10873 177745 : = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
10874 : op);
10875 : }
10876 178172 : gimple_seq_add_stmt (&ctor_seq, init_stmt);
10877 178172 : op = new_temp;
10878 : }
10879 : }
10880 1806710 : elts[number_of_places_left_in_vector] = op;
10881 1806710 : if (!CONSTANT_CLASS_P (op))
10882 322146 : constant_p = false;
10883 : /* For BB vectorization we have to compute an insert location
10884 : when a def is inside the analyzed region since we cannot
10885 : simply insert at the BB start in this case. */
10886 1806710 : stmt_vec_info opdef;
10887 1806710 : if (TREE_CODE (orig_op) == SSA_NAME
10888 184927 : && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
10889 164718 : && is_a <bb_vec_info> (vinfo)
10890 1913647 : && (opdef = vinfo->lookup_def (orig_op)))
10891 : {
10892 87177 : if (!insert_after)
10893 : insert_after = opdef;
10894 : else
10895 47933 : insert_after = get_later_stmt (insert_after, opdef);
10896 : }
10897 :
10898 1806710 : if (number_of_places_left_in_vector == 0)
10899 : {
10900 629340 : auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
10901 629340 : if (uniform_elt)
10902 653744 : vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
10903 326872 : elts[0]);
10904 604936 : else if (constant_p
10905 604936 : ? multiple_p (type_nunits, nunits)
10906 111775 : : known_eq (type_nunits, nunits))
10907 302468 : vec_cst = gimple_build_vector (&ctor_seq, &elts);
10908 : else
10909 : {
10910 0 : if (permute_results.is_empty ())
10911 0 : duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
10912 : elts, number_of_vectors,
10913 : permute_results);
10914 0 : vec_cst = permute_results[number_of_vectors - j - 1];
10915 : }
10916 629340 : if (!gimple_seq_empty_p (ctor_seq))
10917 : {
10918 139523 : if (insert_after)
10919 : {
10920 39244 : gimple_stmt_iterator gsi;
10921 39244 : if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
10922 : {
10923 698 : gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
10924 698 : gsi_insert_seq_before (&gsi, ctor_seq,
10925 : GSI_CONTINUE_LINKING);
10926 : }
10927 38546 : else if (!stmt_ends_bb_p (insert_after->stmt))
10928 : {
10929 38546 : gsi = gsi_for_stmt (insert_after->stmt);
10930 38546 : gsi_insert_seq_after (&gsi, ctor_seq,
10931 : GSI_CONTINUE_LINKING);
10932 : }
10933 : else
10934 : {
10935 : /* When we want to insert after a def where the
10936 : defining stmt throws then insert on the fallthru
10937 : edge. */
10938 0 : edge e = find_fallthru_edge
10939 0 : (gimple_bb (insert_after->stmt)->succs);
10940 0 : basic_block new_bb
10941 0 : = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
10942 0 : gcc_assert (!new_bb);
10943 : }
10944 : }
10945 : else
10946 100279 : vinfo->insert_seq_on_entry (NULL, ctor_seq);
10947 139523 : ctor_seq = NULL;
10948 : }
10949 629340 : voprnds.quick_push (vec_cst);
10950 629340 : insert_after = NULL;
10951 629340 : number_of_places_left_in_vector = nunits;
10952 629340 : constant_p = true;
10953 629340 : elts.new_vector (vector_type, nunits, 1);
10954 629340 : elts.quick_grow (nunits);
10955 : }
10956 : }
10957 : }
10958 :
10959 : /* Since the vectors are created in the reverse order, we should invert
10960 : them. */
10961 498514 : vec_num = voprnds.length ();
10962 1127854 : for (j = vec_num; j != 0; j--)
10963 : {
10964 629340 : vop = voprnds[j - 1];
10965 629340 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
10966 : }
10967 :
10968 : /* In case that VF is greater than the unrolling factor needed for the SLP
10969 : group of stmts, NUMBER_OF_VECTORS to be created is greater than
10970 : NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
10971 : to replicate the vectors. */
10972 498514 : while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
10973 498514 : for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
10974 : i++)
10975 0 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
10976 498514 : }
10977 :
10978 : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
10979 : if there is no definition for it in the scalar IL or it is not known. */
10980 :
10981 : tree
10982 2617 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
10983 : {
10984 2617 : if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
10985 : {
10986 2605 : if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
10987 : return NULL_TREE;
10988 2605 : stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
10989 2605 : if (!def)
10990 : return NULL_TREE;
10991 2605 : return gimple_get_lhs (STMT_VINFO_STMT (def));
10992 : }
10993 : else
10994 12 : return SLP_TREE_SCALAR_OPS (slp_node)[n];
10995 : }
10996 :
10997 : /* Get the Ith vectorized definition from SLP_NODE. */
10998 :
10999 : tree
11000 147771 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
11001 : {
11002 147771 : return SLP_TREE_VEC_DEFS (slp_node)[i];
11003 : }
11004 :
11005 : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
11006 :
11007 : void
11008 947664 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
11009 : {
11010 1895328 : vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
11011 947664 : vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
11012 947664 : }
11013 :
11014 : /* Get N vectorized definitions for SLP_NODE. */
11015 :
11016 : void
11017 2953 : vect_get_slp_defs (vec_info *,
11018 : slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
11019 : {
11020 2953 : if (n == -1U)
11021 2953 : n = SLP_TREE_CHILDREN (slp_node).length ();
11022 :
11023 10648 : for (unsigned i = 0; i < n; ++i)
11024 : {
11025 7695 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
11026 7695 : vec<tree> vec_defs = vNULL;
11027 7695 : vect_get_slp_defs (child, &vec_defs);
11028 7695 : vec_oprnds->quick_push (vec_defs);
11029 : }
11030 2953 : }
11031 :
11032 : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
11033 : - PERM gives the permutation that the caller wants to use for NODE,
11034 : which might be different from SLP_LOAD_PERMUTATION.
11035 : - DUMP_P controls whether the function dumps information. */
11036 :
11037 : static bool
11038 137914 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
11039 : load_permutation_t &perm,
11040 : const vec<tree> &dr_chain,
11041 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11042 : bool analyze_only, bool dump_p,
11043 : unsigned *n_perms, unsigned int *n_loads,
11044 : bool dce_chain)
11045 : {
11046 137914 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
11047 137914 : int vec_index = 0;
11048 137914 : tree vectype = SLP_TREE_VECTYPE (node);
11049 137914 : unsigned int group_size = SLP_TREE_LANES (node);
11050 137914 : unsigned int mask_element;
11051 137914 : unsigned dr_group_size;
11052 137914 : machine_mode mode;
11053 :
11054 137914 : if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
11055 : {
11056 : /* We have both splats of the same non-grouped load and groups
11057 : of distinct invariant loads entering here. */
11058 1491 : unsigned max_idx = 0;
11059 8261 : for (auto idx : perm)
11060 3788 : max_idx = idx > max_idx ? idx : max_idx;
11061 1491 : dr_group_size = max_idx + 1;
11062 : }
11063 : else
11064 : {
11065 136423 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
11066 136423 : dr_group_size = DR_GROUP_SIZE (stmt_info);
11067 : }
11068 :
11069 137914 : mode = TYPE_MODE (vectype);
11070 137914 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11071 137914 : unsigned int nstmts = vect_get_num_copies (vinfo, node);
11072 :
11073 : /* Initialize the vect stmts of NODE to properly insert the generated
11074 : stmts later. */
11075 137914 : if (! analyze_only)
11076 58773 : for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
11077 22623 : SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
11078 :
11079 : /* Generate permutation masks for every NODE. Number of masks for each NODE
11080 : is equal to GROUP_SIZE.
11081 : E.g., we have a group of three nodes with three loads from the same
11082 : location in each node, and the vector size is 4. I.e., we have a
11083 : a0b0c0a1b1c1... sequence and we need to create the following vectors:
11084 : for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
11085 : for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
11086 : ...
11087 :
11088 : The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
11089 : The last mask is illegal since we assume two operands for permute
11090 : operation, and the mask element values can't be outside that range.
11091 : Hence, the last mask must be converted into {2,5,5,5}.
11092 : For the first two permutations we need the first and the second input
11093 : vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
11094 : we need the second and the third vectors: {b1,c1,a2,b2} and
11095 : {c2,a3,b3,c3}. */
11096 :
11097 137914 : int vect_stmts_counter = 0;
11098 137914 : unsigned int index = 0;
11099 137914 : int first_vec_index = -1;
11100 137914 : int second_vec_index = -1;
11101 137914 : bool noop_p = true;
11102 137914 : *n_perms = 0;
11103 :
11104 137914 : vec_perm_builder mask;
11105 137914 : unsigned int nelts_to_build;
11106 137914 : unsigned int nvectors_per_build;
11107 137914 : unsigned int in_nlanes;
11108 137914 : bool repeating_p = (group_size == dr_group_size
11109 173509 : && multiple_p (nunits, group_size));
11110 137914 : if (repeating_p)
11111 : {
11112 : /* A single vector contains a whole number of copies of the node, so:
11113 : (a) all permutes can use the same mask; and
11114 : (b) the permutes only need a single vector input. */
11115 33187 : mask.new_vector (nunits, group_size, 3);
11116 33187 : nelts_to_build = mask.encoded_nelts ();
11117 : /* It's possible to obtain zero nstmts during analyze_only, so make
11118 : it at least one to ensure the later computation for n_perms
11119 : proceed. */
11120 33187 : nvectors_per_build = nstmts > 0 ? nstmts : 1;
11121 33187 : in_nlanes = dr_group_size * 3;
11122 : }
11123 : else
11124 : {
11125 : /* We need to construct a separate mask for each vector statement. */
11126 104727 : unsigned HOST_WIDE_INT const_nunits, const_vf;
11127 104727 : if (!nunits.is_constant (&const_nunits)
11128 104727 : || !vf.is_constant (&const_vf))
11129 : return false;
11130 104727 : mask.new_vector (const_nunits, const_nunits, 1);
11131 104727 : nelts_to_build = const_vf * group_size;
11132 104727 : nvectors_per_build = 1;
11133 104727 : in_nlanes = const_vf * dr_group_size;
11134 : }
11135 137914 : auto_sbitmap used_in_lanes (in_nlanes);
11136 137914 : bitmap_clear (used_in_lanes);
11137 137914 : auto_bitmap used_defs;
11138 :
11139 137914 : unsigned int count = mask.encoded_nelts ();
11140 137914 : mask.quick_grow (count);
11141 137914 : vec_perm_indices indices;
11142 :
11143 723912 : for (unsigned int j = 0; j < nelts_to_build; j++)
11144 : {
11145 597373 : unsigned int iter_num = j / group_size;
11146 597373 : unsigned int stmt_num = j % group_size;
11147 597373 : unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
11148 597373 : bitmap_set_bit (used_in_lanes, i);
11149 597373 : if (repeating_p)
11150 : {
11151 : first_vec_index = 0;
11152 : mask_element = i;
11153 : }
11154 : else
11155 : {
11156 : /* Enforced before the loop when !repeating_p. */
11157 381163 : unsigned int const_nunits = nunits.to_constant ();
11158 381163 : vec_index = i / const_nunits;
11159 381163 : mask_element = i % const_nunits;
11160 381163 : if (vec_index == first_vec_index
11161 381163 : || first_vec_index == -1)
11162 : {
11163 : first_vec_index = vec_index;
11164 : }
11165 152744 : else if (vec_index == second_vec_index
11166 152744 : || second_vec_index == -1)
11167 : {
11168 146475 : second_vec_index = vec_index;
11169 146475 : mask_element += const_nunits;
11170 : }
11171 : else
11172 : {
11173 6269 : if (dump_p)
11174 280 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11175 : "permutation requires at "
11176 : "least three vectors %G",
11177 : stmt_info->stmt);
11178 6269 : gcc_assert (analyze_only);
11179 : return false;
11180 : }
11181 :
11182 374894 : gcc_assert (mask_element < 2 * const_nunits);
11183 : }
11184 :
11185 591104 : if (mask_element != index)
11186 380343 : noop_p = false;
11187 591104 : mask[index++] = mask_element;
11188 :
11189 591104 : if (index == count)
11190 : {
11191 162369 : if (!noop_p)
11192 : {
11193 132287 : indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
11194 132287 : if (!can_vec_perm_const_p (mode, mode, indices))
11195 : {
11196 5106 : if (dump_p)
11197 : {
11198 79 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11199 : "unsupported vect permute { ");
11200 673 : for (i = 0; i < count; ++i)
11201 : {
11202 594 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11203 594 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11204 : }
11205 79 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11206 : }
11207 5106 : gcc_assert (analyze_only);
11208 : return false;
11209 : }
11210 :
11211 127181 : tree mask_vec = NULL_TREE;
11212 127181 : if (!analyze_only)
11213 20904 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11214 :
11215 127181 : if (second_vec_index == -1)
11216 36553 : second_vec_index = first_vec_index;
11217 :
11218 257236 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11219 : {
11220 130055 : ++*n_perms;
11221 130055 : if (analyze_only)
11222 108868 : continue;
11223 : /* Generate the permute statement if necessary. */
11224 21187 : tree first_vec = dr_chain[first_vec_index + ri];
11225 21187 : tree second_vec = dr_chain[second_vec_index + ri];
11226 21187 : gassign *stmt = as_a<gassign *> (stmt_info->stmt);
11227 21187 : tree perm_dest
11228 21187 : = vect_create_destination_var (gimple_assign_lhs (stmt),
11229 : vectype);
11230 21187 : perm_dest = make_ssa_name (perm_dest);
11231 21187 : gimple *perm_stmt
11232 21187 : = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
11233 : second_vec, mask_vec);
11234 21187 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
11235 : gsi);
11236 21187 : if (dce_chain)
11237 : {
11238 20264 : bitmap_set_bit (used_defs, first_vec_index + ri);
11239 20264 : bitmap_set_bit (used_defs, second_vec_index + ri);
11240 : }
11241 :
11242 : /* Store the vector statement in NODE. */
11243 21187 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
11244 : }
11245 : }
11246 30082 : else if (!analyze_only)
11247 : {
11248 2872 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11249 : {
11250 1436 : tree first_vec = dr_chain[first_vec_index + ri];
11251 : /* If mask was NULL_TREE generate the requested
11252 : identity transform. */
11253 1436 : if (dce_chain)
11254 1429 : bitmap_set_bit (used_defs, first_vec_index + ri);
11255 :
11256 : /* Store the vector statement in NODE. */
11257 1436 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
11258 : }
11259 : }
11260 :
11261 : index = 0;
11262 : first_vec_index = -1;
11263 : second_vec_index = -1;
11264 : noop_p = true;
11265 : }
11266 : }
11267 :
11268 126539 : if (n_loads)
11269 : {
11270 88779 : if (repeating_p)
11271 10731 : *n_loads = nstmts;
11272 : else
11273 : {
11274 : /* Enforced above when !repeating_p. */
11275 78048 : unsigned int const_nunits = nunits.to_constant ();
11276 78048 : *n_loads = 0;
11277 78048 : bool load_seen = false;
11278 1051973 : for (unsigned i = 0; i < in_nlanes; ++i)
11279 : {
11280 973925 : if (i % const_nunits == 0)
11281 : {
11282 416120 : if (load_seen)
11283 123900 : *n_loads += 1;
11284 : load_seen = false;
11285 : }
11286 973925 : if (bitmap_bit_p (used_in_lanes, i))
11287 270362 : load_seen = true;
11288 : }
11289 78048 : if (load_seen)
11290 51748 : *n_loads += 1;
11291 : }
11292 : }
11293 :
11294 126539 : if (dce_chain)
11295 229345 : for (unsigned i = 0; i < dr_chain.length (); ++i)
11296 74149 : if (!bitmap_bit_p (used_defs, i))
11297 : {
11298 40891 : tree def = dr_chain[i];
11299 41301 : do
11300 : {
11301 41301 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11302 41301 : if (is_gimple_assign (stmt)
11303 41301 : && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
11304 41301 : || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
11305 4979 : def = single_ssa_tree_operand (stmt, SSA_OP_USE);
11306 : else
11307 : def = NULL;
11308 41301 : gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
11309 41301 : gsi_remove (&rgsi, true);
11310 41301 : release_defs (stmt);
11311 : }
11312 41301 : while (def);
11313 : }
11314 :
11315 : return true;
11316 137914 : }
11317 :
11318 : /* Generate vector permute statements from a list of loads in DR_CHAIN.
11319 : If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
11320 : permute statements for the SLP node NODE. Store the number of vector
11321 : permute instructions in *N_PERMS and the number of vector load
11322 : instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions
11323 : that were not needed. */
11324 :
11325 : bool
11326 98580 : vect_transform_slp_perm_load (vec_info *vinfo,
11327 : slp_tree node, const vec<tree> &dr_chain,
11328 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11329 : bool analyze_only, unsigned *n_perms,
11330 : unsigned int *n_loads, bool dce_chain)
11331 : {
11332 98580 : return vect_transform_slp_perm_load_1 (vinfo, node,
11333 98580 : SLP_TREE_LOAD_PERMUTATION (node),
11334 : dr_chain, gsi, vf, analyze_only,
11335 : dump_enabled_p (), n_perms, n_loads,
11336 98580 : dce_chain);
11337 : }
11338 :
11339 : /* Produce the next vector result for SLP permutation NODE by adding a vector
11340 : statement at GSI. If MASK_VEC is nonnull, add:
11341 :
11342 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
11343 :
11344 : otherwise add:
11345 :
11346 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
11347 : { N, N+1, N+2, ... }>
11348 :
11349 : where N == IDENTITY_OFFSET which is either zero or equal to the
11350 : number of elements of the result. */
11351 :
11352 : static void
11353 31527 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11354 : slp_tree node, tree first_def, tree second_def,
11355 : tree mask_vec, poly_uint64 identity_offset)
11356 : {
11357 31527 : tree vectype = SLP_TREE_VECTYPE (node);
11358 :
11359 : /* ??? We SLP match existing vector element extracts but
11360 : allow punning which we need to re-instantiate at uses
11361 : but have no good way of explicitly representing. */
11362 31527 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
11363 31527 : && !types_compatible_p (TREE_TYPE (first_def), vectype))
11364 : {
11365 20 : gassign *conv_stmt
11366 20 : = gimple_build_assign (make_ssa_name (vectype),
11367 : build1 (VIEW_CONVERT_EXPR, vectype, first_def));
11368 20 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11369 20 : first_def = gimple_assign_lhs (conv_stmt);
11370 : }
11371 31527 : gassign *perm_stmt;
11372 31527 : if (mask_vec)
11373 : {
11374 28125 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
11375 28125 : TYPE_SIZE (vectype))
11376 28125 : && !types_compatible_p (TREE_TYPE (second_def), vectype))
11377 : {
11378 8 : gassign *conv_stmt
11379 8 : = gimple_build_assign (make_ssa_name (vectype),
11380 : build1 (VIEW_CONVERT_EXPR,
11381 : vectype, second_def));
11382 8 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11383 8 : second_def = gimple_assign_lhs (conv_stmt);
11384 : }
11385 28125 : tree perm_dest = make_ssa_name (vectype);
11386 28125 : perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
11387 : first_def, second_def,
11388 : mask_vec);
11389 : }
11390 : else
11391 : {
11392 3402 : auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
11393 3402 : unsigned HOST_WIDE_INT vecno;
11394 3402 : poly_uint64 eltno;
11395 3402 : if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
11396 : &vecno, &eltno))
11397 : gcc_unreachable ();
11398 3402 : tree def = vecno & 1 ? second_def : first_def;
11399 3402 : if (!types_compatible_p (TREE_TYPE (def), vectype))
11400 : {
11401 : /* For identity permutes we still need to handle the case
11402 : of offsetted extracts or concats. */
11403 324 : tree perm_dest = make_ssa_name (vectype);
11404 324 : unsigned HOST_WIDE_INT c;
11405 324 : if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
11406 : {
11407 320 : unsigned HOST_WIDE_INT elsz
11408 320 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
11409 640 : tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
11410 320 : TYPE_SIZE (vectype),
11411 320 : bitsize_int (eltno * elsz));
11412 320 : perm_stmt = gimple_build_assign (perm_dest, lowpart);
11413 : }
11414 4 : else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
11415 4 : def_nunits, &c) && c == 2)
11416 : {
11417 4 : gcc_assert (known_eq (identity_offset, 0U));
11418 4 : tree ctor = build_constructor_va (vectype, 2,
11419 : NULL_TREE, first_def,
11420 : NULL_TREE, second_def);
11421 4 : perm_stmt = gimple_build_assign (perm_dest, ctor);
11422 : }
11423 : else
11424 0 : gcc_unreachable ();
11425 : }
11426 : else
11427 : {
11428 3078 : gcc_assert (known_eq (eltno, 0U));
11429 3078 : node->push_vec_def (def);
11430 3078 : return;
11431 : }
11432 : }
11433 28449 : vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
11434 : /* Store the vector statement in NODE. */
11435 28449 : node->push_vec_def (perm_stmt);
11436 : }
11437 :
11438 : /* Subroutine of vectorizable_slp_permutation. Check whether the target
11439 : can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
11440 : If GSI is nonnull, emit the permutation there.
11441 :
11442 : When GSI is null, the only purpose of NODE is to give properties
11443 : of the result, such as the vector type and number of SLP lanes.
11444 : The node does not need to be a VEC_PERM_EXPR.
11445 :
11446 : If the target supports the operation, return the number of individual
11447 : VEC_PERM_EXPRs needed, otherwise return -1. Print information to the
11448 : dump file if DUMP_P is true. */
11449 :
11450 : static int
11451 454504 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
11452 : slp_tree node, lane_permutation_t &perm,
11453 : vec<slp_tree> &children, bool dump_p)
11454 : {
11455 454504 : tree vectype = SLP_TREE_VECTYPE (node);
11456 :
11457 : /* ??? We currently only support all same vector input types
11458 : while the SLP IL should really do a concat + select and thus accept
11459 : arbitrary mismatches. */
11460 454504 : slp_tree child;
11461 454504 : unsigned i;
11462 454504 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11463 454504 : bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
11464 : /* True if we're permuting a single input of 2N vectors down
11465 : to N vectors. This case doesn't generalize beyond 2 since
11466 : VEC_PERM_EXPR only takes 2 inputs. */
11467 454504 : bool pack_p = false;
11468 : /* If we're permuting inputs of N vectors each into X*N outputs,
11469 : this is the value of X, otherwise it is 1. */
11470 454504 : unsigned int unpack_factor = 1;
11471 454504 : tree op_vectype = NULL_TREE;
11472 456068 : FOR_EACH_VEC_ELT (children, i, child)
11473 455990 : if (SLP_TREE_VECTYPE (child))
11474 : {
11475 : op_vectype = SLP_TREE_VECTYPE (child);
11476 : break;
11477 : }
11478 454504 : if (!op_vectype)
11479 78 : op_vectype = vectype;
11480 954060 : FOR_EACH_VEC_ELT (children, i, child)
11481 : {
11482 499556 : if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
11483 18953 : && !vect_maybe_update_slp_op_vectype (child, op_vectype))
11484 499556 : || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
11485 999112 : || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
11486 : {
11487 0 : if (dump_p)
11488 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11489 : "Unsupported vector types in lane permutation\n");
11490 0 : return -1;
11491 : }
11492 499556 : auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
11493 499556 : unsigned int this_unpack_factor;
11494 : /* Detect permutations of external, pre-existing vectors. The external
11495 : node's SLP_TREE_LANES stores the total number of units in the vector,
11496 : or zero if the vector has variable length.
11497 :
11498 : We are expected to keep the original VEC_PERM_EXPR for such cases.
11499 : There is no repetition to model. */
11500 499556 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def
11501 499556 : && SLP_TREE_SCALAR_OPS (child).is_empty ())
11502 : repeating_p = false;
11503 : /* Check whether the input has twice as many lanes per vector. */
11504 483171 : else if (children.length () == 1
11505 483171 : && known_eq (SLP_TREE_LANES (child) * nunits,
11506 : SLP_TREE_LANES (node) * op_nunits * 2))
11507 : pack_p = true;
11508 : /* Check whether the output has N times as many lanes per vector. */
11509 499556 : else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
11510 439354 : SLP_TREE_LANES (child) * nunits,
11511 : &this_unpack_factor)
11512 404509 : && (i == 0 || unpack_factor == this_unpack_factor))
11513 : unpack_factor = this_unpack_factor;
11514 : else
11515 : repeating_p = false;
11516 : }
11517 :
11518 909008 : gcc_assert (perm.length () == SLP_TREE_LANES (node));
11519 :
11520 : /* Load-lanes permute. This permute only acts as a forwarder to
11521 : select the correct vector def of the load-lanes load which
11522 : has the permuted vectors in its vector defs like
11523 : { v0, w0, r0, v1, w1, r1 ... } for a ld3. All costs are
11524 : accounted for in the costing for the actual load so we
11525 : return zero here. */
11526 454504 : if (node->ldst_lanes)
11527 : {
11528 0 : gcc_assert (children.length () == 1);
11529 0 : if (!gsi)
11530 : /* This is a trivial op always supported. */
11531 : return 0;
11532 0 : slp_tree child = children[0];
11533 0 : unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
11534 0 : / SLP_TREE_LANES (node));
11535 0 : unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
11536 0 : unsigned nvectors = vect_get_num_copies (vinfo, node);
11537 0 : for (unsigned i = 0; i < nvectors; ++i)
11538 : {
11539 0 : tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num + vec_idx];
11540 0 : node->push_vec_def (def);
11541 : }
11542 : return 0;
11543 : }
11544 :
11545 : /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
11546 : and if we can generate the vectors in a vector-length agnostic way.
11547 : This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
11548 : compile time.
11549 :
11550 : The significance of UNPACK_STEP is that, when PACK_P is false,
11551 : output vector I operates on a window of UNPACK_STEP elements from each
11552 : input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR). For example,
11553 : when UNPACK_FACTOR is 2, the first output vector operates on lanes
11554 : [0, NUNITS / 2 - 1] of each input vector and the second output vector
11555 : operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
11556 :
11557 : When REPEATING_P is true, NOUTPUTS holds the total number of outputs
11558 : that we actually need to generate. */
11559 454504 : uint64_t noutputs = 0;
11560 454504 : poly_uint64 unpack_step = 0;
11561 454504 : loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
11562 182935 : if (!linfo
11563 493546 : || !multiple_p (nunits, unpack_factor, &unpack_step)
11564 181995 : || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
11565 181995 : * SLP_TREE_LANES (node), nunits, &noutputs))
11566 : repeating_p = false;
11567 :
11568 : /* We can handle the conditions described for REPEATING_P above for
11569 : both variable- and constant-length vectors. The fallback requires
11570 : us to generate every element of every permute vector explicitly,
11571 : which is only possible for constant-length permute vectors.
11572 :
11573 : Set:
11574 :
11575 : - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
11576 : mask vectors that we want to build.
11577 :
11578 : - NCOPIES to the number of copies of PERM that we need in order
11579 : to build the necessary permute mask vectors. */
11580 181995 : uint64_t npatterns;
11581 181995 : unsigned nelts_per_pattern;
11582 181995 : uint64_t ncopies;
11583 181995 : if (repeating_p)
11584 : {
11585 : /* We need permute mask vectors that have the form:
11586 :
11587 : { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
11588 :
11589 : In other words, the original n-element permute in PERM is
11590 : "unrolled" to fill a full vector. The stepped vector encoding
11591 : that we use for permutes requires 3n elements. */
11592 142953 : npatterns = SLP_TREE_LANES (node);
11593 142953 : nelts_per_pattern = ncopies = 3;
11594 : }
11595 : else
11596 : {
11597 : /* Calculate every element of every permute mask vector explicitly,
11598 : instead of relying on the pattern described above. */
11599 311551 : if (!nunits.is_constant (&npatterns)
11600 311551 : || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
11601 : {
11602 : if (dump_p)
11603 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11604 : "unsupported permutation %p on variable-length"
11605 : " vectors\n", (void *) node);
11606 : return -1;
11607 : }
11608 311551 : nelts_per_pattern = ncopies = 1;
11609 311551 : if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
11610 : {
11611 : if (dump_p)
11612 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11613 : "unsupported permutation %p for variable VF\n",
11614 : (void *) node);
11615 : return -1;
11616 : }
11617 : pack_p = false;
11618 : unpack_factor = 1;
11619 : }
11620 454504 : unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
11621 454504 : gcc_assert (repeating_p || multiple_p (olanes, nunits));
11622 :
11623 : /* Compute the { { SLP operand, vector index}, lane } permutation sequence
11624 : from the { SLP operand, scalar lane } permutation as recorded in the
11625 : SLP node as intermediate step. This part should already work
11626 : with SLP children with arbitrary number of lanes. */
11627 454504 : auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
11628 454504 : auto_vec<poly_uint64> active_lane;
11629 454504 : vperm.create (olanes);
11630 454504 : active_lane.safe_grow_cleared (children.length (), true);
11631 917287 : for (unsigned int ui = 0; ui < unpack_factor; ++ui)
11632 : {
11633 1957746 : for (unsigned j = 0; j < children.length (); ++j)
11634 516090 : active_lane[j] = ui * unpack_step;
11635 1325969 : for (unsigned i = 0; i < ncopies; ++i)
11636 : {
11637 5437966 : for (unsigned pi = 0; pi < perm.length (); ++pi)
11638 : {
11639 1855797 : std::pair<unsigned, unsigned> p = perm[pi];
11640 1855797 : tree vtype = SLP_TREE_VECTYPE (children[p.first]);
11641 1855797 : if (repeating_p)
11642 834045 : vperm.quick_push ({{p.first, 0},
11643 834045 : p.second + active_lane[p.first]});
11644 : else
11645 : {
11646 : /* We checked above that the vectors are constant-length. */
11647 1021752 : unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
11648 1021752 : .to_constant ();
11649 1021752 : unsigned lane = active_lane[p.first].to_constant ();
11650 1021752 : unsigned vi = (lane + p.second) / vnunits;
11651 1021752 : unsigned vl = (lane + p.second) % vnunits;
11652 1021752 : vperm.quick_push ({{p.first, vi}, vl});
11653 : }
11654 : }
11655 : /* Advance to the next group. */
11656 1844107 : for (unsigned j = 0; j < children.length (); ++j)
11657 980921 : active_lane[j] += SLP_TREE_LANES (children[j]);
11658 : }
11659 : }
11660 :
11661 454504 : if (dump_p)
11662 : {
11663 8963 : dump_printf_loc (MSG_NOTE, vect_location,
11664 : "vectorizing permutation %p", (void *)node);
11665 32466 : for (unsigned i = 0; i < perm.length (); ++i)
11666 23503 : dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
11667 8963 : if (repeating_p)
11668 7566 : dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
11669 8963 : dump_printf (MSG_NOTE, "\n");
11670 8963 : dump_printf_loc (MSG_NOTE, vect_location, "as");
11671 90454 : for (unsigned i = 0; i < vperm.length (); ++i)
11672 : {
11673 81491 : if (i != 0
11674 81491 : && (repeating_p
11675 55311 : ? multiple_p (i, npatterns)
11676 60629 : : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
11677 24361 : dump_printf (MSG_NOTE, ",");
11678 81491 : dump_printf (MSG_NOTE, " vops%u[%u][",
11679 81491 : vperm[i].first.first, vperm[i].first.second);
11680 81491 : dump_dec (MSG_NOTE, vperm[i].second);
11681 81491 : dump_printf (MSG_NOTE, "]");
11682 : }
11683 8963 : dump_printf (MSG_NOTE, "\n");
11684 : }
11685 :
11686 : /* We can only handle two-vector permutes, everything else should
11687 : be lowered on the SLP level. The following is closely inspired
11688 : by vect_transform_slp_perm_load and is supposed to eventually
11689 : replace it.
11690 : ??? As intermediate step do code-gen in the SLP tree representation
11691 : somehow? */
11692 454504 : std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
11693 454504 : std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
11694 454504 : unsigned int index = 0;
11695 454504 : poly_uint64 mask_element;
11696 454504 : vec_perm_builder mask;
11697 454504 : mask.new_vector (nunits, npatterns, nelts_per_pattern);
11698 454504 : unsigned int count = mask.encoded_nelts ();
11699 454504 : mask.quick_grow (count);
11700 454504 : vec_perm_indices indices;
11701 454504 : unsigned nperms = 0;
11702 : /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
11703 : vectors to check during analysis, but we need to generate NOUTPUTS
11704 : vectors during transformation. */
11705 454504 : unsigned total_nelts = olanes;
11706 454504 : unsigned process_nelts = olanes;
11707 454504 : if (repeating_p)
11708 : {
11709 142953 : total_nelts = (total_nelts / unpack_factor) * noutputs;
11710 142953 : if (gsi)
11711 9815 : process_nelts = total_nelts;
11712 : }
11713 454504 : unsigned last_ei = (total_nelts - 1) % process_nelts;
11714 2313027 : for (unsigned i = 0; i < process_nelts; ++i)
11715 : {
11716 : /* VI is the input vector index when generating code for REPEATING_P. */
11717 1869139 : unsigned vi = i / olanes * (pack_p ? 2 : 1);
11718 1869139 : unsigned ei = i % olanes;
11719 1869139 : mask_element = vperm[ei].second;
11720 1869139 : if (pack_p)
11721 : {
11722 : /* In this case, we have N outputs and the single child provides 2N
11723 : inputs. Output X permutes inputs 2X and 2X+1.
11724 :
11725 : The mask indices are taken directly from the SLP permutation node.
11726 : Index X selects from the first vector if (X / NUNITS) % 2 == 0;
11727 : X selects from the second vector otherwise. These conditions
11728 : are only known at compile time for constant-length vectors. */
11729 : first_vec = std::make_pair (0, 0);
11730 : second_vec = std::make_pair (0, 1);
11731 : }
11732 1699783 : else if (first_vec.first == -1U
11733 1699783 : || first_vec == vperm[ei].first)
11734 1505600 : first_vec = vperm[ei].first;
11735 194183 : else if (second_vec.first == -1U
11736 194183 : || second_vec == vperm[ei].first)
11737 : {
11738 193796 : second_vec = vperm[ei].first;
11739 193796 : mask_element += nunits;
11740 : }
11741 : else
11742 : {
11743 387 : if (dump_p)
11744 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11745 : "permutation requires at "
11746 : "least three vectors\n");
11747 387 : gcc_assert (!gsi);
11748 : return -1;
11749 : }
11750 :
11751 1868752 : mask[index++] = mask_element;
11752 :
11753 1868752 : if (index == count)
11754 : {
11755 597579 : indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
11756 : TYPE_VECTOR_SUBPARTS (op_vectype));
11757 597579 : bool identity_p = (indices.series_p (0, 1, mask[0], 1)
11758 932815 : && constant_multiple_p (mask[0], nunits));
11759 597579 : machine_mode vmode = TYPE_MODE (vectype);
11760 597579 : machine_mode op_vmode = TYPE_MODE (op_vectype);
11761 597579 : unsigned HOST_WIDE_INT c;
11762 597579 : if ((!identity_p
11763 549856 : && !can_vec_perm_const_p (vmode, op_vmode, indices))
11764 597579 : || (identity_p
11765 47723 : && !known_le (nunits,
11766 : TYPE_VECTOR_SUBPARTS (op_vectype))
11767 10237 : && (!constant_multiple_p (nunits,
11768 8 : TYPE_VECTOR_SUBPARTS (op_vectype),
11769 8 : &c) || c != 2)))
11770 : {
11771 10229 : if (dump_p)
11772 : {
11773 154 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
11774 : vect_location,
11775 : "unsupported vect permute { ");
11776 1596 : for (i = 0; i < count; ++i)
11777 : {
11778 1442 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11779 1442 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11780 : }
11781 154 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11782 : }
11783 10229 : gcc_assert (!gsi);
11784 10616 : return -1;
11785 : }
11786 :
11787 587350 : if (!identity_p)
11788 539627 : nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
11789 587350 : if (gsi)
11790 : {
11791 31527 : if (second_vec.first == -1U)
11792 7271 : second_vec = first_vec;
11793 :
11794 31527 : slp_tree
11795 31527 : first_node = children[first_vec.first],
11796 31527 : second_node = children[second_vec.first];
11797 :
11798 31527 : tree mask_vec = NULL_TREE;
11799 31527 : if (!identity_p)
11800 28125 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11801 :
11802 31527 : tree first_def
11803 31527 : = vect_get_slp_vect_def (first_node, first_vec.second + vi);
11804 31527 : tree second_def
11805 31527 : = vect_get_slp_vect_def (second_node, second_vec.second + vi);
11806 31527 : vect_add_slp_permutation (vinfo, gsi, node, first_def,
11807 31527 : second_def, mask_vec, mask[0]);
11808 : }
11809 :
11810 : index = 0;
11811 : first_vec = std::make_pair (-1U, -1U);
11812 : second_vec = std::make_pair (-1U, -1U);
11813 : }
11814 : }
11815 :
11816 443888 : return nperms;
11817 454504 : }
11818 :
11819 : /* Vectorize the SLP permutations in NODE as specified
11820 : in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
11821 : child number and lane number.
11822 : Interleaving of two two-lane two-child SLP subtrees (not supported):
11823 : [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
11824 : A blend of two four-lane two-child SLP subtrees:
11825 : [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
11826 : Highpart of a four-lane one-child SLP subtree (not supported):
11827 : [ { 0, 2 }, { 0, 3 } ]
11828 : Where currently only a subset is supported by code generating below. */
11829 :
11830 : bool
11831 99041 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11832 : slp_tree node, stmt_vector_for_cost *cost_vec)
11833 : {
11834 99041 : tree vectype = SLP_TREE_VECTYPE (node);
11835 99041 : lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
11836 99041 : int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
11837 99041 : SLP_TREE_CHILDREN (node),
11838 : dump_enabled_p ());
11839 99041 : if (nperms < 0)
11840 : return false;
11841 :
11842 97418 : if (!gsi && nperms != 0)
11843 74417 : record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
11844 :
11845 : return true;
11846 : }
11847 :
11848 : /* Vectorize SLP NODE. */
11849 :
11850 : static void
11851 1494661 : vect_schedule_slp_node (vec_info *vinfo,
11852 : slp_tree node, slp_instance instance)
11853 : {
11854 1494661 : gimple_stmt_iterator si;
11855 1494661 : int i;
11856 1494661 : slp_tree child;
11857 :
11858 : /* Vectorize externals and constants. */
11859 1494661 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
11860 1494661 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
11861 : {
11862 : /* ??? vectorizable_shift can end up using a scalar operand which is
11863 : currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
11864 : node in this case. */
11865 506615 : if (!SLP_TREE_VECTYPE (node))
11866 506615 : return;
11867 :
11868 : /* There are two reasons vector defs might already exist. The first
11869 : is that we are vectorizing an existing vector def. The second is
11870 : when performing BB vectorization shared constant/external nodes
11871 : are not split apart during partitioning so during the code-gen
11872 : DFS walk we can end up visiting them twice. */
11873 499444 : if (! SLP_TREE_VEC_DEFS (node).exists ())
11874 498514 : vect_create_constant_vectors (vinfo, node);
11875 499444 : return;
11876 : }
11877 :
11878 988046 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
11879 :
11880 988046 : gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
11881 988046 : if (SLP_TREE_VECTYPE (node))
11882 988040 : SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
11883 :
11884 988046 : if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
11885 : {
11886 : /* Vectorized loads go before the first scalar load to make it
11887 : ready early, vectorized stores go before the last scalar
11888 : stmt which is where all uses are ready. */
11889 722278 : stmt_vec_info last_stmt_info = NULL;
11890 722278 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
11891 169450 : last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
11892 : else /* DR_IS_WRITE */
11893 552828 : last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
11894 722278 : si = gsi_for_stmt (last_stmt_info->stmt);
11895 722278 : }
11896 265768 : else if (!SLP_TREE_PERMUTE_P (node)
11897 249102 : && (SLP_TREE_TYPE (node) == cycle_phi_info_type
11898 : || SLP_TREE_TYPE (node) == induc_vec_info_type
11899 : || SLP_TREE_TYPE (node) == phi_info_type))
11900 : {
11901 : /* For PHI node vectorization we do not use the insertion iterator. */
11902 54684 : si = gsi_none ();
11903 : }
11904 : else
11905 : {
11906 : /* Emit other stmts after the children vectorized defs which is
11907 : earliest possible. */
11908 : gimple *last_stmt = NULL;
11909 586720 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
11910 375636 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
11911 : {
11912 : /* For fold-left reductions we are retaining the scalar
11913 : reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
11914 : set so the representation isn't perfect. Resort to the
11915 : last scalar def here. */
11916 299992 : if (SLP_TREE_VEC_DEFS (child).is_empty ())
11917 : {
11918 940 : gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
11919 940 : gphi *phi = as_a <gphi *>
11920 940 : (vect_find_last_scalar_stmt_in_slp (child)->stmt);
11921 940 : if (!last_stmt)
11922 : last_stmt = phi;
11923 721 : else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
11924 : last_stmt = phi;
11925 710 : else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
11926 : ;
11927 : else
11928 0 : gcc_unreachable ();
11929 : }
11930 : /* We are emitting all vectorized stmts in the same place and
11931 : the last one is the last.
11932 : ??? Unless we have a load permutation applied and that
11933 : figures to re-use an earlier generated load. */
11934 : unsigned j;
11935 : tree vdef;
11936 708432 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
11937 408440 : if (TREE_CODE (vdef) == SSA_NAME
11938 408440 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
11939 : {
11940 408388 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
11941 408388 : if (!last_stmt)
11942 : last_stmt = vstmt;
11943 209038 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
11944 : last_stmt = vstmt;
11945 46642 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
11946 : ;
11947 : else
11948 0 : gcc_unreachable ();
11949 : }
11950 : }
11951 75644 : else if (!SLP_TREE_VECTYPE (child))
11952 : {
11953 : /* For externals we use unvectorized at all scalar defs. */
11954 : unsigned j;
11955 : tree def;
11956 15642 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
11957 : /* If the stmt is not inside the region do not
11958 : use it as possible insertion point. */
11959 9099 : if (auto stmt_info = vinfo->lookup_def (def))
11960 : {
11961 250 : gimple *stmt = stmt_info->stmt;
11962 250 : if (!last_stmt)
11963 : last_stmt = stmt;
11964 230 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
11965 : last_stmt = stmt;
11966 26 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
11967 : ;
11968 : else
11969 0 : gcc_unreachable ();
11970 : }
11971 : }
11972 : else
11973 : {
11974 : /* For externals we have to look at all defs since their
11975 : insertion place is decided per vector. But beware
11976 : of pre-existing vectors where we need to make sure
11977 : we do not insert before the region boundary. */
11978 69101 : if (SLP_TREE_SCALAR_OPS (child).is_empty ()
11979 698 : && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
11980 : ;
11981 : else
11982 : {
11983 : unsigned j;
11984 : tree vdef;
11985 541709 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
11986 97103 : if (TREE_CODE (vdef) == SSA_NAME
11987 97103 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
11988 : {
11989 21117 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
11990 21117 : if (!last_stmt)
11991 : last_stmt = vstmt;
11992 11386 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
11993 : last_stmt = vstmt;
11994 8985 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
11995 : ;
11996 : else
11997 0 : gcc_unreachable ();
11998 : }
11999 : }
12000 : }
12001 :
12002 : /* We split regions to vectorize at control altering stmts
12003 : with a definition so this can only be an external. */
12004 211084 : gcc_checking_assert (!last_stmt
12005 : || !is_ctrl_altering_stmt (last_stmt));
12006 :
12007 211084 : if (is_a <bb_vec_info> (vinfo)
12008 20835 : && !SLP_TREE_PERMUTE_P (node)
12009 19170 : && (!last_stmt
12010 19150 : || gimple_bb (last_stmt) != gimple_bb (stmt_info->stmt))
12011 212759 : && gimple_could_trap_p (stmt_info->stmt))
12012 : {
12013 : /* We've constrained possibly trapping operations to all come
12014 : from the same basic-block, if vectorized defs would allow earlier
12015 : scheduling still force vectorized stmts to the original block.
12016 : This is only necessary for BB vectorization since for loop vect
12017 : all operations are in a single BB and scalar stmt based
12018 : placement doesn't play well with epilogue vectorization. */
12019 137 : gcc_assert (!last_stmt
12020 : || dominated_by_p (CDI_DOMINATORS,
12021 : gimple_bb (stmt_info->stmt),
12022 : gimple_bb (last_stmt)));
12023 137 : si = gsi_after_labels (gimple_bb (stmt_info->stmt));
12024 : }
12025 : /* When there is no in-region child def to guide placement, insert
12026 : at region boundary. */
12027 210947 : else if (!last_stmt)
12028 1756 : si = gsi_after_labels (vinfo->bbs[0]);
12029 209191 : else if (is_a <gphi *> (last_stmt))
12030 14560 : si = gsi_after_labels (gimple_bb (last_stmt));
12031 : else
12032 : {
12033 194631 : si = gsi_for_stmt (last_stmt);
12034 : /* We use gsi_insert_before, so when last_stmt is a vector
12035 : def we have to advance (or use gsi_insert_after). */
12036 194631 : gsi_next (&si);
12037 :
12038 194631 : if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
12039 : {
12040 : /* Avoid scheduling stmts to random places in the CFG, any
12041 : stmt dominance check we performed is possibly wrong as UIDs
12042 : are not initialized for all of the function for loop
12043 : vectorization. Instead append to the loop preheader. */
12044 174344 : if ((LOOP_VINFO_LOOP (loop_vinfo)->header
12045 174344 : != gimple_bb (last_stmt))
12046 177489 : && dominated_by_p (CDI_DOMINATORS,
12047 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12048 3145 : gimple_bb (last_stmt)))
12049 1262 : si = gsi_end_bb (loop_preheader_edge
12050 631 : (LOOP_VINFO_LOOP (loop_vinfo))->src);
12051 : /* Avoid scheduling internal defs outside of the loop when
12052 : we might have only implicitly tracked loop mask/len defs. */
12053 76 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
12054 174344 : || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12055 : {
12056 76 : gimple_stmt_iterator si2
12057 76 : = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
12058 76 : if ((gsi_end_p (si2)
12059 0 : && (LOOP_VINFO_LOOP (loop_vinfo)->header
12060 0 : != gimple_bb (last_stmt))
12061 0 : && dominated_by_p (CDI_DOMINATORS,
12062 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12063 0 : gimple_bb (last_stmt)))
12064 76 : || (!gsi_end_p (si2)
12065 76 : && last_stmt != *si2
12066 73 : && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
12067 3 : si = si2;
12068 : }
12069 : }
12070 : }
12071 : }
12072 :
12073 988046 : if (dump_enabled_p ())
12074 : {
12075 71827 : if (stmt_info)
12076 71775 : dump_printf_loc (MSG_NOTE, vect_location,
12077 : "------>vectorizing SLP node starting from: %G",
12078 : stmt_info->stmt);
12079 : else
12080 : {
12081 52 : dump_printf_loc (MSG_NOTE, vect_location,
12082 : "------>vectorizing SLP node:\n");
12083 52 : vect_print_slp_tree (MSG_NOTE, vect_location, node);
12084 : }
12085 : }
12086 988046 : vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
12087 : }
12088 :
12089 : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
12090 : For loop vectorization this is done in vectorizable_call, but for SLP
12091 : it needs to be deferred until end of vect_schedule_slp, because multiple
12092 : SLP instances may refer to the same scalar stmt. */
12093 :
12094 : static void
12095 604137 : vect_remove_slp_scalar_calls (vec_info *vinfo,
12096 : slp_tree node, hash_set<slp_tree> &visited)
12097 : {
12098 604137 : gimple *new_stmt;
12099 604137 : gimple_stmt_iterator gsi;
12100 604137 : tree lhs;
12101 :
12102 604137 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12103 189094 : return;
12104 :
12105 458708 : if (visited.add (node))
12106 : return;
12107 :
12108 1546891 : for (auto child : SLP_TREE_CHILDREN (node))
12109 513742 : vect_remove_slp_scalar_calls (vinfo, child, visited);
12110 :
12111 1724710 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
12112 : {
12113 487859 : if (!stmt_info)
12114 3978 : continue;
12115 483881 : stmt_info = vect_orig_stmt (stmt_info);
12116 483881 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
12117 5142 : if (!stmt || gimple_bb (stmt) == NULL)
12118 478781 : continue;
12119 5100 : lhs = gimple_call_lhs (stmt);
12120 5100 : if (lhs)
12121 4529 : new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
12122 : else
12123 571 : new_stmt = gimple_build_nop ();
12124 5100 : unlink_stmt_vdef (stmt_info->stmt);
12125 5100 : gsi = gsi_for_stmt (stmt);
12126 5100 : vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
12127 5100 : if (lhs)
12128 4529 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12129 : }
12130 : }
12131 :
12132 : static void
12133 90395 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
12134 : {
12135 90395 : hash_set<slp_tree> visited;
12136 90395 : vect_remove_slp_scalar_calls (vinfo, node, visited);
12137 90395 : }
12138 :
12139 : /* Vectorize the instance root. */
12140 :
12141 : void
12142 13458 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
12143 : {
12144 13458 : gassign *rstmt = NULL;
12145 :
12146 13458 : if (instance->kind == slp_inst_kind_ctor)
12147 : {
12148 5467 : if (SLP_TREE_VEC_DEFS (node).length () == 1)
12149 : {
12150 5422 : tree vect_lhs = SLP_TREE_VEC_DEFS (node)[0];
12151 5422 : tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12152 5422 : if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
12153 5422 : TREE_TYPE (vect_lhs)))
12154 0 : vect_lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
12155 : vect_lhs);
12156 5422 : rstmt = gimple_build_assign (root_lhs, vect_lhs);
12157 : }
12158 : else
12159 : {
12160 45 : gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
12161 45 : tree child_def;
12162 45 : int j;
12163 45 : vec<constructor_elt, va_gc> *v;
12164 45 : vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
12165 :
12166 : /* A CTOR can handle V16HI composition from VNx8HI so we
12167 : do not need to convert vector elements if the types
12168 : do not match. */
12169 135 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
12170 90 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
12171 45 : tree lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12172 45 : tree rtype
12173 45 : = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
12174 45 : tree r_constructor = build_constructor (rtype, v);
12175 45 : rstmt = gimple_build_assign (lhs, r_constructor);
12176 : }
12177 : }
12178 7991 : else if (instance->kind == slp_inst_kind_bb_reduc)
12179 : {
12180 : /* Largely inspired by reduction chain epilogue handling in
12181 : vect_create_epilog_for_reduction. */
12182 6399 : vec<tree> vec_defs = vNULL;
12183 6399 : vect_get_slp_defs (node, &vec_defs);
12184 6399 : enum tree_code reduc_code
12185 6399 : = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
12186 : /* ??? We actually have to reflect signs somewhere. */
12187 6399 : if (reduc_code == MINUS_EXPR)
12188 0 : reduc_code = PLUS_EXPR;
12189 6399 : gimple_seq epilogue = NULL;
12190 : /* We may end up with more than one vector result, reduce them
12191 : to one vector. */
12192 6399 : tree vec_def = vec_defs[0];
12193 6399 : tree vectype = TREE_TYPE (vec_def);
12194 6399 : tree compute_vectype = vectype;
12195 6399 : bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
12196 5275 : && TYPE_OVERFLOW_UNDEFINED (vectype)
12197 9958 : && operation_can_overflow (reduc_code));
12198 3007 : if (pun_for_overflow_p)
12199 : {
12200 3007 : compute_vectype = unsigned_type_for (vectype);
12201 3007 : vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12202 : compute_vectype, vec_def);
12203 : }
12204 8897 : for (unsigned i = 1; i < vec_defs.length (); ++i)
12205 : {
12206 2498 : tree def = vec_defs[i];
12207 2498 : if (pun_for_overflow_p)
12208 2293 : def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12209 : compute_vectype, def);
12210 2498 : vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
12211 : vec_def, def);
12212 : }
12213 6399 : vec_defs.release ();
12214 : /* ??? Support other schemes than direct internal fn or two
12215 : element vectors. */
12216 6399 : tree scalar_def;
12217 6399 : internal_fn reduc_fn;
12218 6399 : if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
12219 6399 : || reduc_fn == IFN_LAST
12220 12798 : || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
12221 : OPTIMIZE_FOR_BOTH))
12222 : {
12223 1633 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
12224 1633 : tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
12225 1633 : TREE_TYPE (compute_vectype), vec_def,
12226 1633 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12227 1633 : bitsize_zero_node);
12228 1633 : tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
12229 1633 : TREE_TYPE (compute_vectype), vec_def,
12230 1633 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12231 1633 : TYPE_SIZE (TREE_TYPE (compute_vectype)));
12232 1633 : scalar_def = gimple_build (&epilogue, reduc_code,
12233 1633 : TREE_TYPE (compute_vectype), tem0, tem1);
12234 : }
12235 : else
12236 4766 : scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
12237 4766 : TREE_TYPE (compute_vectype), vec_def);
12238 6399 : if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
12239 : {
12240 3458 : tree rem_def = NULL_TREE;
12241 14402 : for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
12242 : {
12243 10944 : def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
12244 10944 : if (!rem_def)
12245 : rem_def = def;
12246 : else
12247 7486 : rem_def = gimple_build (&epilogue, reduc_code,
12248 7486 : TREE_TYPE (scalar_def),
12249 : rem_def, def);
12250 : }
12251 3458 : scalar_def = gimple_build (&epilogue, reduc_code,
12252 3458 : TREE_TYPE (scalar_def),
12253 : scalar_def, rem_def);
12254 : }
12255 6399 : scalar_def = gimple_convert (&epilogue,
12256 6399 : TREE_TYPE (vectype), scalar_def);
12257 6399 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12258 6399 : gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
12259 6399 : gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
12260 6399 : update_stmt (gsi_stmt (rgsi));
12261 6399 : return;
12262 : }
12263 1592 : else if (instance->kind == slp_inst_kind_gcond)
12264 : {
12265 : /* Only support a single root for now as we can't codegen CFG yet and so we
12266 : can't support lane > 1 at this time. */
12267 1592 : gcc_assert (instance->root_stmts.length () == 1);
12268 1592 : auto root_stmt_info = instance->root_stmts[0];
12269 1592 : auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
12270 1592 : gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
12271 1592 : gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
12272 1592 : bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
12273 : root_stmt_info, &rgsi, node, NULL);
12274 1592 : gcc_assert (res);
12275 1592 : return;
12276 : }
12277 : else
12278 0 : gcc_unreachable ();
12279 :
12280 5467 : gcc_assert (rstmt);
12281 :
12282 5467 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12283 5467 : gsi_replace (&rgsi, rstmt, true);
12284 : }
12285 :
12286 : struct slp_scc_info
12287 : {
12288 : bool on_stack;
12289 : int dfs;
12290 : int lowlink;
12291 : };
12292 :
12293 : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs. */
12294 :
12295 : static void
12296 1494661 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
12297 : hash_map<slp_tree, slp_scc_info> &scc_info,
12298 : int &maxdfs, vec<slp_tree> &stack)
12299 : {
12300 1494661 : bool existed_p;
12301 1494661 : slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
12302 1494661 : gcc_assert (!existed_p);
12303 1494661 : info->dfs = maxdfs;
12304 1494661 : info->lowlink = maxdfs;
12305 1494661 : maxdfs++;
12306 :
12307 : /* Leaf. */
12308 1494661 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12309 : {
12310 506615 : info->on_stack = false;
12311 506615 : vect_schedule_slp_node (vinfo, node, instance);
12312 1045096 : return;
12313 : }
12314 :
12315 988046 : info->on_stack = true;
12316 988046 : stack.safe_push (node);
12317 :
12318 988046 : unsigned i;
12319 988046 : slp_tree child;
12320 : /* DFS recurse. */
12321 2038306 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12322 : {
12323 1050260 : if (!child)
12324 55513 : continue;
12325 994747 : slp_scc_info *child_info = scc_info.get (child);
12326 994747 : if (!child_info)
12327 : {
12328 903044 : vect_schedule_scc (vinfo, child, instance, scc_info, maxdfs, stack);
12329 : /* Recursion might have re-allocated the node. */
12330 903044 : info = scc_info.get (node);
12331 903044 : child_info = scc_info.get (child);
12332 903044 : info->lowlink = MIN (info->lowlink, child_info->lowlink);
12333 : }
12334 91703 : else if (child_info->on_stack)
12335 25605 : info->lowlink = MIN (info->lowlink, child_info->dfs);
12336 : }
12337 988046 : if (info->lowlink != info->dfs)
12338 : return;
12339 :
12340 956180 : auto_vec<slp_tree, 4> phis_to_fixup;
12341 :
12342 : /* Singleton. */
12343 956180 : if (stack.last () == node)
12344 : {
12345 932250 : stack.pop ();
12346 932250 : info->on_stack = false;
12347 932250 : vect_schedule_slp_node (vinfo, node, instance);
12348 932250 : if (!SLP_TREE_PERMUTE_P (node)
12349 932250 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
12350 30883 : phis_to_fixup.quick_push (node);
12351 : }
12352 : else
12353 : {
12354 : /* SCC. */
12355 23930 : int last_idx = stack.length () - 1;
12356 55796 : while (stack[last_idx] != node)
12357 31866 : last_idx--;
12358 : /* We can break the cycle at PHIs who have at least one child
12359 : code generated. Then we could re-start the DFS walk until
12360 : all nodes in the SCC are covered (we might have new entries
12361 : for only back-reachable nodes). But it's simpler to just
12362 : iterate and schedule those that are ready. */
12363 23930 : unsigned todo = stack.length () - last_idx;
12364 24271 : do
12365 : {
12366 106130 : for (int idx = stack.length () - 1; idx >= last_idx; --idx)
12367 : {
12368 57588 : slp_tree entry = stack[idx];
12369 57588 : if (!entry)
12370 958 : continue;
12371 56630 : bool phi = (!SLP_TREE_PERMUTE_P (entry)
12372 56630 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
12373 56630 : bool ready = !phi;
12374 143285 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
12375 111835 : if (!child)
12376 : {
12377 23029 : gcc_assert (phi);
12378 : ready = true;
12379 : break;
12380 : }
12381 88806 : else if (scc_info.get (child)->on_stack)
12382 : {
12383 24111 : if (!phi)
12384 : {
12385 : ready = false;
12386 : break;
12387 : }
12388 : }
12389 : else
12390 : {
12391 64695 : if (phi)
12392 : {
12393 : ready = true;
12394 : break;
12395 : }
12396 : }
12397 33601 : if (ready)
12398 : {
12399 55796 : vect_schedule_slp_node (vinfo, entry, instance);
12400 55796 : scc_info.get (entry)->on_stack = false;
12401 55796 : stack[idx] = NULL;
12402 55796 : todo--;
12403 55796 : if (phi)
12404 24376 : phis_to_fixup.safe_push (entry);
12405 : }
12406 : }
12407 : }
12408 24271 : while (todo != 0);
12409 :
12410 : /* Pop the SCC. */
12411 23930 : stack.truncate (last_idx);
12412 : }
12413 :
12414 : /* Now fixup the backedge def of the vectorized PHIs in this SCC. */
12415 : slp_tree phi_node;
12416 1967619 : FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
12417 : {
12418 55259 : gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
12419 55259 : edge_iterator ei;
12420 55259 : edge e;
12421 174657 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
12422 : {
12423 119398 : unsigned dest_idx = e->dest_idx;
12424 119398 : child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
12425 119398 : if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
12426 66758 : continue;
12427 52640 : unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
12428 : /* Simply fill all args. */
12429 52640 : if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
12430 : != vect_first_order_recurrence)
12431 113127 : for (unsigned i = 0; i < n; ++i)
12432 : {
12433 60532 : tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
12434 60532 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
12435 60532 : add_phi_arg (phi, vect_get_slp_vect_def (child, i),
12436 : e, gimple_phi_arg_location (phi, dest_idx));
12437 : }
12438 : else
12439 : {
12440 : /* Unless it is a first order recurrence which needs
12441 : args filled in for both the PHI node and the permutes. */
12442 45 : gimple *perm
12443 45 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
12444 45 : gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
12445 45 : add_phi_arg (as_a <gphi *> (rphi),
12446 : vect_get_slp_vect_def (child, n - 1),
12447 : e, gimple_phi_arg_location (phi, dest_idx));
12448 127 : for (unsigned i = 0; i < n; ++i)
12449 : {
12450 82 : gimple *perm
12451 82 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
12452 82 : if (i > 0)
12453 37 : gimple_assign_set_rhs1 (perm,
12454 : vect_get_slp_vect_def (child, i - 1));
12455 82 : gimple_assign_set_rhs2 (perm,
12456 : vect_get_slp_vect_def (child, i));
12457 82 : update_stmt (perm);
12458 : }
12459 : }
12460 : }
12461 : }
12462 956180 : }
12463 :
12464 : /* Generate vector code for SLP_INSTANCES in the loop/basic block. */
12465 :
12466 : void
12467 551583 : vect_schedule_slp (vec_info *vinfo, const vec<slp_instance> &slp_instances)
12468 : {
12469 551583 : slp_instance instance;
12470 551583 : unsigned int i;
12471 :
12472 551583 : hash_map<slp_tree, slp_scc_info> scc_info;
12473 551583 : int maxdfs = 0;
12474 1143333 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12475 : {
12476 591750 : slp_tree node = SLP_INSTANCE_TREE (instance);
12477 591750 : if (dump_enabled_p ())
12478 : {
12479 16097 : dump_printf_loc (MSG_NOTE, vect_location,
12480 : "Vectorizing SLP tree:\n");
12481 : /* ??? Dump all? */
12482 16097 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12483 485 : dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
12484 485 : SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
12485 16097 : vect_print_slp_graph (MSG_NOTE, vect_location,
12486 : SLP_INSTANCE_TREE (instance));
12487 : }
12488 : /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
12489 : have a PHI be the node breaking the cycle. */
12490 591750 : auto_vec<slp_tree> stack;
12491 591750 : if (!scc_info.get (node))
12492 591617 : vect_schedule_scc (vinfo, node, instance, scc_info, maxdfs, stack);
12493 :
12494 591750 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12495 13458 : vectorize_slp_instance_root_stmt (vinfo, node, instance);
12496 :
12497 591750 : if (dump_enabled_p ())
12498 16097 : dump_printf_loc (MSG_NOTE, vect_location,
12499 : "vectorizing stmts using SLP.\n");
12500 591750 : }
12501 :
12502 1694916 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12503 : {
12504 591750 : slp_tree root = SLP_INSTANCE_TREE (instance);
12505 591750 : stmt_vec_info store_info;
12506 591750 : unsigned int j;
12507 :
12508 : /* Remove scalar call stmts. Do not do this for basic-block
12509 : vectorization as not all uses may be vectorized.
12510 : ??? Why should this be necessary? DCE should be able to
12511 : remove the stmts itself.
12512 : ??? For BB vectorization we can as well remove scalar
12513 : stmts starting from the SLP tree root if they have no
12514 : uses. */
12515 591750 : if (is_a <loop_vec_info> (vinfo))
12516 90395 : vect_remove_slp_scalar_calls (vinfo, root);
12517 :
12518 : /* Remove vectorized stores original scalar stmts. */
12519 2635564 : for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
12520 : {
12521 1490986 : if (!store_info
12522 1490972 : || !STMT_VINFO_DATA_REF (store_info)
12523 1461654 : || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
12524 : break;
12525 :
12526 1452064 : store_info = vect_orig_stmt (store_info);
12527 : /* Free the attached stmt_vec_info and remove the stmt. */
12528 1452064 : vinfo->remove_stmt (store_info);
12529 :
12530 : /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
12531 : to not crash in vect_free_slp_tree later. */
12532 1452064 : if (SLP_TREE_REPRESENTATIVE (root) == store_info)
12533 552489 : SLP_TREE_REPRESENTATIVE (root) = NULL;
12534 : }
12535 : }
12536 551583 : }
|
